Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

prov/efa: Add unit tests for efa_msg #10632

Merged
merged 1 commit into from
Dec 13, 2024
Merged

prov/efa: Add unit tests for efa_msg #10632

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion prov/efa/Makefile.include
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,8 @@ nodist_prov_efa_test_efa_unit_test_SOURCES = \
prov/efa/test/efa_unit_test_runt.c \
prov/efa/test/efa_unit_test_mr.c \
prov/efa/test/efa_unit_test_rdm_peer.c \
prov/efa/test/efa_unit_test_pke.c
prov/efa/test/efa_unit_test_pke.c \
prov/efa/test/efa_unit_test_msg.c


efa_CPPFLAGS += -I$(top_srcdir)/include -I$(top_srcdir)/prov/efa/test $(cmocka_CPPFLAGS)
Expand Down
12 changes: 12 additions & 0 deletions prov/efa/test/efa_unit_test_mocks.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,12 @@ void efa_mock_ibv_wr_send_verify_handshake_pkt_local_host_id_and_save_wr(struct
return efa_mock_ibv_wr_send_save_wr(qp);
}

void efa_mock_ibv_wr_send_imm_save_wr(struct ibv_qp_ex *qp, __be32 imm_data)
{
g_ibv_submitted_wr_id_vec[g_ibv_submitted_wr_id_cnt] = (void *)qp->wr_id;
g_ibv_submitted_wr_id_cnt++;
}

void efa_mock_ibv_wr_set_inline_data_list_no_op(struct ibv_qp_ex *qp,
size_t num_buf,
const struct ibv_data_buf *buf_list)
Expand Down Expand Up @@ -207,6 +213,12 @@ bool efa_mock_efa_device_support_unsolicited_write_recv()
return mock();
}

int efa_mock_ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr,
struct ibv_recv_wr **bad_wr)
{
return mock();
}

struct efa_unit_test_mocks g_efa_unit_test_mocks = {
.local_host_id = 0,
.peer_host_id = 0,
Expand Down
5 changes: 5 additions & 0 deletions prov/efa/test/efa_unit_test_mocks.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ uint32_t efa_mock_ibv_read_wc_flags_return_mock(struct ibv_cq_ex *current);

bool efa_mock_efadv_wc_is_unsolicited(struct efadv_cq *efadv_cq);

void efa_mock_ibv_wr_send_imm_save_wr(struct ibv_qp_ex *qp, __be32 imm_data);

ssize_t __real_ofi_copy_from_hmem_iov(void *dest, size_t size,
enum fi_hmem_iface hmem_iface, uint64_t device,
const struct iovec *hmem_iov,
Expand All @@ -93,6 +95,9 @@ int efa_mock_efa_rdm_pke_read_return_mock(struct efa_rdm_ope *ope);

bool efa_mock_efa_device_support_unsolicited_write_recv(void);

int efa_mock_ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr,
struct ibv_recv_wr **bad_wr);

struct efa_unit_test_mocks
{
uint64_t local_host_id;
Expand Down
267 changes: 267 additions & 0 deletions prov/efa/test/efa_unit_test_msg.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,267 @@
/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */
/* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All
* rights reserved. */

#include "efa_unit_tests.h"
#include "ofi_util.h"

extern struct fi_ops_msg efa_msg_ops;

static void test_efa_msg_recv_prep(struct efa_resource *resource,
fi_addr_t *addr)
{
struct ibv_qp *ibv_qp;
struct efa_ep_addr raw_addr;
struct efa_base_ep *base_ep;
size_t raw_addr_len = sizeof(raw_addr);
int ret;

efa_unit_test_resource_construct(resource, FI_EP_RDM);
resource->ep->msg = &efa_msg_ops;

base_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
ibv_qp = base_ep->qp->ibv_qp;
ibv_qp->context->ops.post_recv = &efa_mock_ibv_post_recv;
will_return(efa_mock_ibv_post_recv, 0);

ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
assert_int_equal(ret, 0);
raw_addr.qpn = 1;
raw_addr.qkey = 0x1234;
ret = fi_av_insert(resource->av, &raw_addr, 1, addr, 0 /* flags */,
NULL /* context */);
assert_int_equal(ret, 1);
}

void test_efa_msg_fi_recv(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_unit_test_buff send_buff;
fi_addr_t addr;
int ret;
void *desc;

test_efa_msg_recv_prep(resource, &addr);
efa_unit_test_buff_construct(&send_buff, resource, 4096 /* buff_size */);

desc = fi_mr_desc(send_buff.mr);

ret = fi_recv(resource->ep, send_buff.buff, send_buff.size, desc, addr,
NULL /* context */);
assert_int_equal(ret, 0);

efa_unit_test_buff_destruct(&send_buff);
}

void test_efa_msg_fi_recvv(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_unit_test_buff send_buff;
struct iovec iov;
fi_addr_t addr;
int ret;
void *desc;

test_efa_msg_recv_prep(resource, &addr);
efa_unit_test_buff_construct(&send_buff, resource, 4096 /* buff_size */);

iov.iov_base = send_buff.buff;
iov.iov_len = send_buff.size;
desc = fi_mr_desc(send_buff.mr);

ret = fi_recvv(resource->ep, &iov, &desc, 1, addr, NULL /* context */);
assert_int_equal(ret, 0);

efa_unit_test_buff_destruct(&send_buff);
}

void test_efa_msg_fi_recvmsg(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_unit_test_buff send_buff;
fi_addr_t addr;
int ret;
void *desc;
struct iovec iov;
struct fi_msg msg = {0};

test_efa_msg_recv_prep(resource, &addr);
efa_unit_test_buff_construct(&send_buff, resource, 4096 /* buff_size */);

iov.iov_base = send_buff.buff;
iov.iov_len = send_buff.size;
desc = fi_mr_desc(send_buff.mr);
efa_unit_test_construct_msg(&msg, &iov, 1, addr, NULL, 0, &desc);

ret = fi_recvmsg(resource->ep, &msg, 0);
assert_int_equal(ret, 0);

efa_unit_test_buff_destruct(&send_buff);
}

static void test_efa_msg_send_prep(struct efa_resource *resource,
fi_addr_t *addr)
{
struct ibv_qp_ex *ibv_qpx;
struct efa_ep_addr raw_addr;
struct efa_base_ep *base_ep;
size_t raw_addr_len = sizeof(raw_addr);
int ret;

efa_unit_test_resource_construct(resource, FI_EP_RDM);
resource->ep->msg = &efa_msg_ops;

base_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
ibv_qpx = base_ep->qp->ibv_qp_ex;

ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
assert_int_equal(ret, 0);
raw_addr.qpn = 1;
raw_addr.qkey = 0x1234;
ret = fi_av_insert(resource->av, &raw_addr, 1, addr, 0 /* flags */,
NULL /* context */);
assert_int_equal(ret, 1);

ibv_qpx->wr_start = &efa_mock_ibv_wr_start_no_op;
/* this mock will save the send work request (wr) in a global list */
ibv_qpx->wr_send = &efa_mock_ibv_wr_send_save_wr;
ibv_qpx->wr_send_imm = &efa_mock_ibv_wr_send_imm_save_wr;
ibv_qpx->wr_set_inline_data_list = &efa_mock_ibv_wr_set_inline_data_list_no_op;
ibv_qpx->wr_set_sge_list = &efa_mock_ibv_wr_set_sge_list_no_op;
ibv_qpx->wr_set_ud_addr = &efa_mock_ibv_wr_set_ud_addr_no_op;
ibv_qpx->wr_complete = &efa_mock_ibv_wr_complete_no_op;
}

void test_efa_msg_fi_send(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_unit_test_buff send_buff;
fi_addr_t addr;
void *desc;
int ret;

test_efa_msg_send_prep(resource, &addr);
efa_unit_test_buff_construct(&send_buff, resource, 4096 /* buff_size */);

desc = fi_mr_desc(send_buff.mr);

assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
ret = fi_send(resource->ep, send_buff.buff, send_buff.size, desc, addr,
NULL /* context */);
assert_int_equal(ret, 0);
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);

efa_unit_test_buff_destruct(&send_buff);
}

void test_efa_msg_fi_sendv(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_unit_test_buff send_buff;
fi_addr_t addr;
struct iovec iov;
void *desc;
int ret;

test_efa_msg_send_prep(resource, &addr);
efa_unit_test_buff_construct(&send_buff, resource, 4096 /* buff_size */);

iov.iov_base = send_buff.buff;
iov.iov_len = send_buff.size;
desc = fi_mr_desc(send_buff.mr);

assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
ret = fi_sendv(resource->ep, &iov, &desc, 1, addr, NULL);
assert_int_equal(ret, 0);
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);

efa_unit_test_buff_destruct(&send_buff);
}

void test_efa_msg_fi_sendmsg(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_unit_test_buff send_buff;
fi_addr_t addr;
struct iovec iov;
void *desc;
int ret;
struct fi_msg msg = {0};

test_efa_msg_send_prep(resource, &addr);
efa_unit_test_buff_construct(&send_buff, resource, 4096 /* buff_size */);

iov.iov_base = send_buff.buff;
iov.iov_len = send_buff.size;
desc = fi_mr_desc(send_buff.mr);

efa_unit_test_construct_msg(&msg, &iov, 1, addr, NULL, 0, &desc);

assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
ret = fi_sendmsg(resource->ep, &msg, 0);
assert_int_equal(ret, 0);
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);

efa_unit_test_buff_destruct(&send_buff);
}

void test_efa_msg_fi_senddata(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_unit_test_buff send_buff;
fi_addr_t addr;
void *desc;
int ret;
uint64_t data = 0x1234567890ABCDEF;

test_efa_msg_send_prep(resource, &addr);
efa_unit_test_buff_construct(&send_buff, resource, 4096 /* buff_size */);

desc = fi_mr_desc(send_buff.mr);

assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
ret = fi_senddata(resource->ep, send_buff.buff, send_buff.size, desc,
data, addr, NULL);
assert_int_equal(ret, 0);
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);

efa_unit_test_buff_destruct(&send_buff);
}

void test_efa_msg_fi_inject(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_unit_test_buff send_buff;
fi_addr_t addr;
int ret;

test_efa_msg_send_prep(resource, &addr);
efa_unit_test_buff_construct(&send_buff, resource, 32);

assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
ret = fi_inject(resource->ep, send_buff.buff, send_buff.size, addr);
assert_int_equal(ret, 0);
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);

efa_unit_test_buff_destruct(&send_buff);
}

void test_efa_msg_fi_injectdata(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_unit_test_buff send_buff;
fi_addr_t addr;
int ret;
uint64_t data = 0x1234567890ABCDEF;

test_efa_msg_send_prep(resource, &addr);
efa_unit_test_buff_construct(&send_buff, resource, 32);

assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
ret = fi_injectdata(resource->ep, send_buff.buff, send_buff.size, data,
addr);
assert_int_equal(ret, 0);
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);

efa_unit_test_buff_destruct(&send_buff);
}
9 changes: 9 additions & 0 deletions prov/efa/test/efa_unit_tests.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,15 @@ int main(void)
cmocka_unit_test_setup_teardown(test_efa_rdm_peer_keep_pke_in_overflow_list, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_efa_rdm_peer_append_overflow_pke_to_recvwin, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_efa_rdm_pke_handle_longcts_rtm_send_completion, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_efa_msg_fi_recv, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_efa_msg_fi_recvv, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_efa_msg_fi_recvmsg, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_efa_msg_fi_send, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_efa_msg_fi_sendv, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_efa_msg_fi_sendmsg, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_efa_msg_fi_senddata, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want to add tests for inject* calls?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure how to test the inject as I am using FI_EP_DGRAM as the ep type right now.

Copy link
Contributor

@shijin-aws shijin-aws Dec 12, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

easy thing: use FI_EP_RDM, but override your ep function table before you making the send/recv call.

https://github.com/ofiwg/libfabric/blob/main/prov/efa/src/rdm/efa_rdm_ep_fiops.c#L639C2-L640C32

// get ep from your ep_fid
(*ep)->msg = &efa_msg_ops;
(*ep)->rma = &efa_rma_ops;

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should finally make DGRAM support inject after onboarding FI_CONTEXT2, you can do this override before that happens.

Copy link
Contributor

@shijin-aws shijin-aws Dec 13, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You need to make efa_msg_ops as an external struct for your unit test to declare

cmocka_unit_test_setup_teardown(test_efa_msg_fi_inject, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_efa_msg_fi_injectdata, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
};

cmocka_set_message_output(CM_OUTPUT_XML);
Expand Down
9 changes: 9 additions & 0 deletions prov/efa/test/efa_unit_tests.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,15 @@ void test_efa_rdm_peer_move_overflow_pke_to_recvwin();
void test_efa_rdm_peer_keep_pke_in_overflow_list();
void test_efa_rdm_peer_append_overflow_pke_to_recvwin();
void test_efa_rdm_pke_handle_longcts_rtm_send_completion();
void test_efa_msg_fi_recv();
void test_efa_msg_fi_recvv();
void test_efa_msg_fi_recvmsg();
void test_efa_msg_fi_send();
void test_efa_msg_fi_sendv();
void test_efa_msg_fi_sendmsg();
void test_efa_msg_fi_senddata();
void test_efa_msg_fi_inject();
void test_efa_msg_fi_injectdata();

static inline
int efa_unit_test_get_dlist_length(struct dlist_entry *head)
Expand Down
Loading