Skip to content

Commit

Permalink
Merge pull request ofiwg#678 from aingerson/clean
Browse files Browse the repository at this point in the history 
complex/fi_ubertest: various enhancements/additions to expand testing
  • Loading branch information
@shefty
shefty authored Jul 17, 2017
2 parents 0384ffb + 51e4d58 commit e705c0d
Show file tree
Hide file tree
Showing 26 changed files with 1,432 additions and 315 deletions.
3 changes: 3 additions & 0 deletions Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,13 +264,16 @@ ported_librdmacm_fi_cmatose_LDADD = libfabtests.la

complex_fi_ubertest_SOURCES = \
complex/fabtest.h \
complex/ofi_atomic.h \
complex/ofi_atomic.c \
complex/ft_main.c \
complex/ft_comm.c \
complex/ft_comp.c \
complex/ft_config.c \
complex/ft_domain.c \
complex/ft_endpoint.c \
complex/ft_msg.c \
complex/ft_verify.c \
complex/ft_test.c
complex_fi_ubertest_LDADD = libfabtests.la

Expand Down
62 changes: 55 additions & 7 deletions complex/fabtest.h
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015 Intel Corporation. All rights reserved.
* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
* Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under the BSD license below:
Expand Down Expand Up @@ -85,6 +85,7 @@ struct ft_xcontrol {
fi_addr_t addr;
uint64_t tag;
uint8_t seqno;
uint64_t total_comp;
enum fi_cq_format cq_format;
enum fi_wait_obj comp_wait; /* must be NONE */
uint64_t remote_cq_data;
Expand All @@ -93,6 +94,7 @@ struct ft_xcontrol {
};

struct ft_atomic_control {
void *orig_buf;
void *res_buf;
struct fid_mr *res_mr;
void *res_memdesc;
Expand Down Expand Up @@ -122,6 +124,7 @@ struct ft_control {
int iov_cnt;
int inc_step;
int xfer_iter;
int verify_cnt;
int error;
};

Expand All @@ -132,6 +135,7 @@ extern struct ft_atomic_control ft_atom_ctrl;

enum {
FT_MAX_CAPS = 64,
FT_MAX_CLASS = 4,
FT_MAX_EP_TYPES = 8,
FT_MAX_AV_TYPES = 3,
FT_MAX_PROV_MODES = 4,
Expand All @@ -143,14 +147,15 @@ enum {
enum ft_comp_type {
FT_COMP_UNSPEC,
FT_COMP_QUEUE,
// FT_COMP_COUNTER,
FT_COMP_CNTR,
FT_MAX_COMP
};

enum ft_test_type {
FT_TEST_UNSPEC,
FT_TEST_LATENCY,
FT_TEST_BANDWIDTH,
FT_TEST_UNIT,
FT_MAX_TEST
};

Expand All @@ -161,6 +166,7 @@ enum ft_class_function {
FT_FUNC_SENDMSG,
FT_FUNC_INJECT,
FT_FUNC_INJECTDATA,
FT_FUNC_SENDDATA,
FT_FUNC_READ,
FT_FUNC_READV,
FT_FUNC_READMSG,
Expand All @@ -180,25 +186,58 @@ enum ft_class_function {
FT_FUNC_COMPARE_ATOMIC,
FT_FUNC_COMPARE_ATOMICV,
FT_FUNC_COMPARE_ATOMICMSG,
FT_MAX_FUNCTIONS
FT_MAX_FUNCTIONS
};

#define FT_FLAG_QUICKTEST (1ULL << 0)

#define is_inject_func(x) (x == FT_FUNC_INJECT || \
x == FT_FUNC_INJECTDATA || \
x == FT_FUNC_INJECT_WRITE || \
x == FT_FUNC_INJECT_WRITEDATA || \
x == FT_FUNC_INJECT_ATOMIC)

#define is_read_func(x) (x == FT_FUNC_READ || \
x == FT_FUNC_READV || \
x == FT_FUNC_READMSG)

#define is_fetch_func(x) (x == FT_FUNC_FETCH_ATOMIC || \
x == FT_FUNC_FETCH_ATOMICV || \
x == FT_FUNC_FETCH_ATOMICMSG)

#define is_compare_func(x) (x == FT_FUNC_COMPARE_ATOMIC || \
x == FT_FUNC_COMPARE_ATOMICV || \
x == FT_FUNC_COMPARE_ATOMICMSG)

#define is_data_func(x) (x == FT_FUNC_SENDDATA || \
x == FT_FUNC_INJECTDATA || \
x == FT_FUNC_WRITEDATA || \
x == FT_FUNC_INJECT_WRITEDATA)

#define is_msg_func(x) (x == FT_FUNC_SENDMSG || \
x == FT_FUNC_WRITEMSG || \
x == FT_FUNC_READMSG || \
x == FT_FUNC_ATOMICMSG || \
x == FT_FUNC_FETCH_ATOMICMSG || \
x == FT_FUNC_COMPARE_ATOMICMSG)

struct ft_set {
char node[FI_NAME_MAX];
char service[FI_NAME_MAX];
char prov_name[FI_NAME_MAX];
enum ft_test_type test_type[FT_MAX_TEST];
enum ft_class_function class_function[FT_MAX_FUNCTIONS];
uint64_t msg_flags;
enum fi_op op[FI_ATOMIC_OP_LAST];
enum fi_datatype datatype[FI_DATATYPE_LAST];
enum fi_ep_type ep_type[FT_MAX_EP_TYPES];
enum fi_av_type av_type[FT_MAX_AV_TYPES];
enum ft_comp_type comp_type[FT_MAX_COMP];
enum fi_wait_obj eq_wait_obj[FT_MAX_WAIT_OBJ];
enum fi_wait_obj cq_wait_obj[FT_MAX_WAIT_OBJ];
uint64_t mode[FT_MAX_PROV_MODES];
uint64_t caps[FT_MAX_CAPS];
uint64_t test_class[FT_MAX_CLASS];
uint64_t constant_caps[FT_MAX_CAPS];
uint64_t test_flags;
};

Expand All @@ -211,23 +250,26 @@ struct ft_series {
int cur_type;
int cur_func;
int cur_op;
int cur_datatype;
int cur_ep;
int cur_av;
int cur_comp;
int cur_eq_wait_obj;
int cur_cq_wait_obj;
int cur_mode;
int cur_caps;
int cur_class;
};

struct ft_info {
enum ft_test_type test_type;
int test_index;
int test_subindex;
enum ft_class_function class_function;
uint64_t msg_flags;
enum fi_op op;
enum fi_datatype datatype;
uint64_t test_flags;
uint64_t test_class;
uint64_t caps;
uint64_t mode;
enum fi_av_type av_type;
Expand All @@ -250,6 +292,7 @@ void fts_start(struct ft_series *series, int index);
void fts_next(struct ft_series *series);
int fts_end(struct ft_series *series, int index);
void fts_cur_info(struct ft_series *series, struct ft_info *info);
int fts_info_is_valid(void);


struct ft_msg {
Expand All @@ -265,7 +308,7 @@ int ft_open_control();
ssize_t ft_get_event(uint32_t *event, void *buf, size_t len,
uint32_t event_check, size_t len_check);
int ft_open_comp();
int ft_bind_comp(struct fid_ep *ep, uint64_t flags);
int ft_bind_comp(struct fid_ep *ep);
int ft_comp_rx(int timeout);
int ft_comp_tx(int timeout);

Expand All @@ -274,10 +317,12 @@ int ft_open_passive();
int ft_enable_comm();
int ft_post_recv_bufs();
void ft_format_iov(struct iovec *iov, size_t cnt, char *buf, size_t len);
void ft_format_iocs(struct iovec *iov);
void ft_format_iocs(struct iovec *iov, size_t *iov_count);
void ft_next_iov_cnt(struct ft_xcontrol *ctrl, size_t max_iov_cnt);
int ft_get_ctx(struct ft_xcontrol *ctrl, struct fi_context **ctx);

int ft_send_sync_msg();
int ft_recv_n_msg();
int ft_recv_msg();
int ft_send_msg();
int ft_send_dgram();
Expand All @@ -292,6 +337,9 @@ int ft_run_test();
int ft_reset_ep();
void ft_record_error(int error);

int ft_verify_bufs();
int ft_sync_fill_bufs(size_t size);
void ft_verify_comp(void *buf);

#ifdef __cplusplus
}
Expand Down
110 changes: 87 additions & 23 deletions complex/ft_comp.c
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2013-2015 Intel Corporation. All rights reserved.
* Copyright (c) 2013-2017 Intel Corporation. All rights reserved.
*
* This software is available to you under the BSD license below:
*
Expand Down Expand Up @@ -50,6 +50,24 @@ static size_t comp_entry_size[] = {
};
*/

static int ft_open_cntrs(void)
{
int ret;

opts.comp_method = test_info.cq_wait_obj;

if (!txcntr) {
ret = ft_cntr_open(&txcntr);
if (ret)
return ret;
}
if (!rxcntr) {
ret = ft_cntr_open(&rxcntr);
if (ret)
return ret;
}
return 0;
}

static int ft_open_cqs(void)
{
Expand Down Expand Up @@ -89,37 +107,48 @@ int ft_open_comp(void)
{
int ret;

ret = (test_info.comp_type == FT_COMP_QUEUE) ?
ft_open_cqs() : -FI_ENOSYS;
switch (test_info.comp_type) {
case FT_COMP_QUEUE:
ret = ft_open_cqs();
break;
case FT_COMP_CNTR:
ret = ft_open_cntrs();
break;
default:
ret = -FI_ENOSYS;
}

return ret;
}

int ft_bind_comp(struct fid_ep *ep, uint64_t flags)
int ft_bind_comp(struct fid_ep *ep)
{
int ret;

if (flags & FI_SEND) {
ret = fi_ep_bind(ep, &txcq->fid, flags & ~FI_RECV);
if (ret) {
FT_PRINTERR("fi_ep_bind", ret);
return ret;
}
uint64_t flags;
struct fid *txfid = (test_info.comp_type == FT_COMP_CNTR) ? &txcntr->fid : &txcq->fid;
struct fid *rxfid = (test_info.comp_type == FT_COMP_CNTR) ? &rxcntr->fid : &rxcq->fid;

flags = FI_TRANSMIT;
if (test_info.comp_type == FT_COMP_CNTR)
flags |= FI_READ | FI_WRITE;
ret = fi_ep_bind(ep, txfid, flags);
if (ret) {
FT_PRINTERR("fi_ep_bind", ret);
return ret;
}

if (flags & FI_RECV) {
ret = fi_ep_bind(ep, &rxcq->fid, flags & ~FI_SEND);
if (ret) {
FT_PRINTERR("fi_ep_bind", ret);
return ret;
}
flags = FI_RECV;
ret = fi_ep_bind(ep, rxfid, flags);
if (ret) {
FT_PRINTERR("fi_ep_bind", ret);
return ret;
}

return 0;
}

/* Read CQ until there are no more completions */
#define ft_cq_read(cq_read, cq, buf, count, completions, str, ret, ...) \
#define ft_cq_read(cq_read, cq, buf, count, completions, str, ret, verify,...) \
do { \
ret = cq_read(cq, buf, count, ##__VA_ARGS__); \
if (ret < 0) { \
Expand All @@ -133,6 +162,8 @@ int ft_bind_comp(struct fid_ep *ep, uint64_t flags)
return ret; \
} else { \
completions += ret; \
if (verify) \
ft_verify_comp(buf); \
} \
} while (ret == count)

Expand All @@ -142,7 +173,7 @@ static int ft_comp_x(struct fid_cq *cq, struct ft_xcontrol *ft_x,
uint8_t buf[FT_COMP_BUF_SIZE];
struct timespec s, e;
int poll_time = 0;
int ret;
int ret, verify = (test_info.test_type == FT_TEST_UNIT && cq == rxcq);

switch(test_info.cq_wait_obj) {
case FI_WAIT_NONE:
Expand All @@ -151,7 +182,7 @@ static int ft_comp_x(struct fid_cq *cq, struct ft_xcontrol *ft_x,
clock_gettime(CLOCK_MONOTONIC, &s);

ft_cq_read(fi_cq_read, cq, buf, comp_entry_cnt[ft_x->cq_format],
ft_x->credits, x_str, ret);
ft_x->credits, x_str, ret, verify);

clock_gettime(CLOCK_MONOTONIC, &e);
poll_time = get_elapsed(&s, &e, MILLI);
Expand All @@ -162,7 +193,7 @@ static int ft_comp_x(struct fid_cq *cq, struct ft_xcontrol *ft_x,
case FI_WAIT_FD:
case FI_WAIT_MUTEX_COND:
ft_cq_read(fi_cq_sread, cq, buf, comp_entry_cnt[ft_x->cq_format],
ft_x->credits, x_str, ret, NULL, timeout);
ft_x->credits, x_str, ret, verify, NULL, timeout);
break;
case FI_WAIT_SET:
FT_ERR("fi_ubertest: Unsupported cq wait object");
Expand All @@ -175,13 +206,46 @@ static int ft_comp_x(struct fid_cq *cq, struct ft_xcontrol *ft_x,
return (ret == -FI_EAGAIN && timeout) ? ret : 0;
}

static int ft_cntr_x(struct fid_cntr *cntr, struct ft_xcontrol *ft_x,
int timeout)
{
uint64_t cntr_val;
struct timespec s, e;
int poll_time = clock_gettime(CLOCK_MONOTONIC, &s);

do {
cntr_val = fi_cntr_read(cntr);
clock_gettime(CLOCK_MONOTONIC, &e);
poll_time = get_elapsed(&s, &e, MILLI);
} while (cntr_val == ft_x->total_comp && poll_time < timeout);

ft_x->credits += (cntr_val - ft_x->total_comp);
ft_x->total_comp = cntr_val;

return 0;
}

int ft_comp_rx(int timeout)
{
return ft_comp_x(rxcq, &ft_rx_ctrl, "rxcq", timeout);
switch (test_info.comp_type) {
case FT_COMP_CNTR:
return ft_cntr_x(rxcntr, &ft_rx_ctrl, timeout);
case FT_COMP_QUEUE:
return ft_comp_x(rxcq, &ft_rx_ctrl, "rxcq", timeout);
default:
return -FI_ENOSYS;
}
}


int ft_comp_tx(int timeout)
{
return ft_comp_x(txcq, &ft_tx_ctrl, "txcq", timeout);
switch (test_info.comp_type) {
case FT_COMP_CNTR:
return ft_cntr_x(txcntr, &ft_tx_ctrl, timeout);
case FT_COMP_QUEUE:
return ft_comp_x(txcq, &ft_tx_ctrl, "txcq", timeout);
default:
return -FI_ENOSYS;
}
}
Loading

0 comments on commit e705c0d

Please sign in to comment.