From a93fdce4a62130bdc59a3b2d981df376ca41f719 Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Tue, 20 Apr 2021 16:33:35 -0500 Subject: [PATCH 01/13] 1. Merge the group attribute test code to vfd_swmr_group_writer.c. Typical tests are fine. Need to clean up code and add comments/descriptions. 2. Add group attribute tests to testvfdswmr.sh.in. Only use typical cases for the standard and the quick runs for saving time. --- test/testvfdswmr.sh.in | 88 +- test/vfd_swmr_group_writer.c | 2903 +++++++++++++++++++++++++++++++--- 2 files changed, 2804 insertions(+), 187 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index 28860ca05ac..e287e79c26b 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -54,17 +54,20 @@ fi BIGSET_n=25 # -n option: # of iterations BIGSET_few_s=20 # -s option: # of datasets (for few_big test) BIGSET_many_s=500 # -s option: # of datasets (for many_small test) -GROUP_n=100 # -n option: # of groups (for vfd_swmr_group_writer.c) +GROUP_n=40 # -n option: # of groups (for group test) +GROUP_attr_n=1 # -n option: # of groups (for group attribute test) + if [[ "$HDF5TestExpress" -eq 0 ]] ; then # Setting for exhaustive run BIGSET_n=50 BIGSET_few_s=40 BIGSET_many_s=1000 GROUP_n=400 + GROUP_attr_n=10 elif [[ "$HDF5TestExpress" -gt 1 ]]; then # Setting for quick run BIGSET_n=10 BIGSET_few_s=10 BIGSET_many_s=100 - GROUP_n=40 + GROUP_n=20 fi ############################################################################### @@ -147,7 +150,9 @@ if [ $rc -ne 0 ] ; then fi all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo groups" -all_tests="${all_tests} few_big many_small" +all_tests="${all_tests} groups_attrs few_big many_small" +#all_tests="groups groups_attrs" +#all_tests="groups_attrs" tests=${all_tests} if [ $# -gt 0 ]; then @@ -612,7 +617,7 @@ if [ ${do_zoo:-no} = yes ]; then fi # -# Make sure that we can create GROUP_n groups (40, 100, or 400 depending on the HDF5TestExpress level) +# Make sure that we can create GROUP_n groups (20, 40, or 400 depending on the HDF5TestExpress level) # while a reader waits for each to appear. # if [ ${do_groups:-no} = yes ]; then @@ -649,6 +654,81 @@ if [ ${do_groups:-no} = yes ]; then rm -f vfd_swmr_group_reader.*.{out,rc} fi +grp_attr_list=( + "compact" + "dense" + "compact-to-dense" + "compact-del" + "dense-del" + "dense-del-to-compact" + "modify" + "vstr" + "remove-vstr" + "modify-vstr" + ) +grp_sub_attr_list=( + "dense-del-to-compact" + "modify" + "vstr" + "remove-vstr" + "modify-vstr" + ) + +grp_short_sub_attr_list=( + "dense" + "compact-del" + "modify" + "vstr" + "remove-vstr" + "modify-vstr" + ) + +echo "$HDF5TestExpress" +if [[ "$HDF5TestExpress" -eq 1 ]] ; then #Setting for standard run + grp_attr_list=("${grp_sub_attr_list[@]}") +elif [[ "$HDF5TestExpress" -gt 1 ]] ; then #Setting for quick run + grp_attr_list=("${grp_short_sub_attr_list[@]}") +fi + +for options in ${grp_attr_list[*]}; do + if [ ${do_groups_attrs:-no} = no ]; then + continue + fi + echo launch vfd_swmr_group attribute: $options + catch_out_err_and_rc vfd_swmr_group_writer \ + ../vfd_swmr_group_writer -c 1 -n $GROUP_attr_n -a 1 -A $options & + #../vfd_swmr_group_writer -q -c 1 -n $GROUP_attr_n -a 1 -A $options & + pid_writer=$! + + catch_out_err_and_rc vfd_swmr_group_reader \ + ../vfd_swmr_group_reader -c 1 -n $GROUP_attr_n -a 1 -A $options & + #../vfd_swmr_group_reader -q -c 1 -n $GROUP_attr_n -a 1 -A $options & + pid_reader=$! + + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. + wait $pid_reader + wait $pid_writer + + # Collect exit code of the reader + if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + + # Collect exit code of the writer + if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_group_writer.{out,rc} + rm -f vfd_swmr_group_reader.*.{out,rc} +done + for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do if [ ${do_many_small:-no} = no ]; then continue diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 25ee8c54087..b695b8335db 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -24,17 +24,26 @@ #ifndef H5_HAVE_WIN32_API #define READER_WAIT_TICKS 3 +#define VS_ATTR_NAME_LEN 21 typedef struct { - hid_t file, filetype, one_by_one_sid; - char filename[PATH_MAX]; - char progname[PATH_MAX]; - unsigned int asteps; - unsigned int csteps; - unsigned int nsteps; - unsigned int update_interval; - bool use_vfd_swmr; - bool use_named_pipes; + hid_t file, filetype, one_by_one_sid; + char filename[PATH_MAX]; + char progname[PATH_MAX]; + unsigned int asteps; + unsigned int csteps; + unsigned int nsteps; + unsigned int update_interval; + bool use_vfd_swmr; + bool use_named_pipes; + char at_pattern; + bool attr_test; + uint32_t max_lag; + uint32_t tick_len; + int np_fd_w_to_r; + int np_fd_r_to_w; + int np_notify; + int np_verify; } state_t; #define ALL_HID_INITIALIZER \ @@ -44,25 +53,34 @@ typedef struct { .filetype = H5T_NATIVE_UINT32, .asteps = 10, .csteps = 10, .nsteps = 100, .update_interval = READER_WAIT_TICKS, \ .use_vfd_swmr = true, \ .use_named_pipes = true \ - } + , .at_pattern = ' ' \ + , .attr_test = false \ + , .tick_len = 4 \ + , .max_lag = 7 \ + , .np_fd_w_to_r = -1 \ + , .np_fd_r_to_w = -1 \ + , .np_notify = 0 \ + , .np_verify = 0 } + +//TODO: add at_pattern description static void usage(const char *progname) { - fprintf(stderr, "usage: %s [-S] [-a steps] [-b] [-c]\n" + fprintf(stderr, "usage: %s [-S] [-a steps] [-b] [-c]\n" " [-n iterations] [-N] [-q] [-u numb_ticks]\n" - "\n" - "-S: do not use VFD SWMR\n" - "-a steps: `steps` between adding attributes\n" - "-b: write data in big-endian byte order\n" - "-c steps: `steps` between communication between the writer and reader\n" - "-n ngroups: the number of groups\n" - "-N: do not use named pipes, mainly for running the writer and reader seperately\n" - "-u numb_tcks: `numb_ticks` for the reader to wait before verification\n" - "-q: silence printouts, few messages\n" - "\n", - progname); - exit(EXIT_FAILURE); + "\n" + "-S: do not use VFD SWMR\n" + "-a steps: `steps` between adding attributes\n" + "-b: write data in big-endian byte order\n" + "-c steps: `steps` between communication between the writer and reader\n" + "-n ngroups: the number of groups\n" + "-N: do not use named pipes, mainly for running the writer and reader seperately\n" + "-u numb_tcks: `numb_ticks` for the reader to wait before verification\n" + "-q: silence printouts, few messages\n" + "\n", + progname); + exit(EXIT_FAILURE); } static bool @@ -71,6 +89,16 @@ state_init(state_t *s, int argc, char **argv) unsigned long tmp; int ch; const hsize_t dims = 1; + +#if 0 + char tfile[PATH_MAX]; + char *end; + + *s = ALL_HID_INITIALIZER; + esnprintf(tfile, sizeof(tfile), "%s", argv[0]); + esnprintf(s->progname, sizeof(s->progname), "%s", HDbasename(tfile)); +#endif + char *tfile = NULL; char * end; @@ -87,7 +115,7 @@ state_init(state_t *s, int argc, char **argv) if (tfile) HDfree(tfile); - while ((ch = getopt(argc, argv, "Sa:bc:n:Nqu:")) != -1) { + while ((ch = getopt(argc, argv, "Sa:bc:n:Nqu:A:")) != -1) { switch (ch) { case 'S': s->use_vfd_swmr = false; @@ -127,6 +155,33 @@ state_init(state_t *s, int argc, char **argv) case 'N': s->use_named_pipes = false; break; + case 'A': + if (strcmp(optarg, "compact") == 0) + s->at_pattern = 'c'; + else if (strcmp(optarg, "dense") == 0) + s->at_pattern = 'd'; + else if (strcmp(optarg, "compact-to-dense") == 0) + s->at_pattern = 't'; + else if (strcmp(optarg, "compact-del") == 0) + s->at_pattern = 'C'; + else if (strcmp(optarg, "dense-del") == 0) + s->at_pattern = 'D'; + else if (strcmp(optarg, "dense-del-to-compact") == 0) + s->at_pattern = 'T'; + else if (strcmp(optarg, "modify") == 0) + s->at_pattern = 'M'; + else if (strcmp(optarg,"vstr") ==0) + s->at_pattern = 'v'; + else if (strcmp(optarg, "remove-vstr") == 0) + s->at_pattern = 'r'; + else if (strcmp(optarg, "modify-vstr") == 0) + s->at_pattern = 'm'; + else { + H5_FAILED(); AT(); + printf("Invalid -A argument \"%s\"", optarg); + goto error; + } + break; case 'q': verbosity = 0; break; @@ -171,179 +226,2627 @@ state_init(state_t *s, int argc, char **argv) error: if (tfile) HDfree(tfile); + return false; +} + + +static bool np_wr_send_receive(state_t *s) { + + unsigned int i; + /* Bump up the value of notify to notice the reader to start to read */ + s->np_notify++; + if (HDwrite(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDwrite failed\n"); + goto error; + } + + /* During the wait, writer makes repeated HDF5 API calls + * to trigger EOT at approximately the correct time */ + for(i = 0; i < s->max_lag + 1; i++) { + decisleep(s->tick_len); + H5E_BEGIN_TRY { + H5Aexists(s->file, "nonexistent"); + } H5E_END_TRY; + } + + /* Receive the same value from the reader and verify it before + * going to the next step */ + (s->np_verify)++; + if (HDread(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0){ + H5_FAILED(); AT(); + printf("HDread failed\n"); + goto error; + } + + if (s->np_notify == -1) { + H5_FAILED(); AT(); + printf("reader failed to verify group or attribute operation.\n"); + goto error; + } + + if (s->np_notify != s->np_verify) { + H5_FAILED(); AT(); + printf("received message %d, expecting %d\n", s->np_notify, s->np_verify); + goto error; + } + + return true; + +error: + return false; + +} + +static bool np_rd_receive(state_t *s) { + + /* The writer should have bumped up the value of notify. + * Do the same with verify and confirm it */ + s->np_verify++; + + /* Receive the notify that the writer bumped up the value */ + if (HDread(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDread failed\n"); + goto error; + } + + if (s->np_notify == -1) { + H5_FAILED(); AT(); + printf("writer failed to create group or carry out an attribute operation.\n"); + goto error; + } + + if (s->np_notify != s->np_verify) { + H5_FAILED(); AT(); + printf("received message %d, expecting %d\n", s->np_notify, s->np_verify); + goto error; + } + + return true; + +error: + return false; +} + +static bool np_rd_send(state_t *s) { + + if (HDwrite(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDwrite failed\n"); + return false; + } + else + return true; +} + +static void np_send_error(state_t *s,bool writer) { + s->np_notify = -1; + if(writer) + HDwrite(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)); + else + HDwrite(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)); +} + +static bool +add_attr(state_t *s, hid_t oid,unsigned int which,unsigned num_attrs,const char*aname_fmt) { + + //char attrname[sizeof("attr-d-9999999999-999")]; + char attrname[VS_ATTR_NAME_LEN]; + //char* attrname_base= "attr-%u-%u"; + unsigned u; + //int i; + unsigned attr_value; + hid_t aid = H5I_INVALID_HID; + hid_t amtype = H5I_INVALID_HID; + hid_t atype = s->filetype; + hid_t sid = s->one_by_one_sid; + +// Just for debugging +#if 0 +if(which == 1) + goto error; +#endif + + if((amtype = H5Tget_native_type(atype,H5T_DIR_ASCEND)) <0) { + H5_FAILED(); AT(); + printf("H5Tget_native_type failed\n"); + goto error; + } + + for (u = 0; u < num_attrs; u++) { + + /* Create attribute */ + //HDsprintf(attrname, "attr-%u-%u", which,u); + //HDsprintf(attrname, attrname_base, which,u); + HDsprintf(attrname, aname_fmt, which,u); + if((aid = H5Acreate2(oid, attrname, atype, sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Acreate2 failed\n"); + goto error; + } + + attr_value = u+which; +#if 0 + // Just for debugging to check if error handling works. + attr_value = u+which+1; +#endif + + dbgf(1, "setting attribute %s on group %u to %u\n", attrname, which, u+which); + /* Write data into the attribute */ + if (H5Awrite(aid, amtype, &attr_value) < 0) { + H5_FAILED(); AT(); + printf("H5Awrite failed\n"); + goto error; + } + + /* Close attribute */ + if(H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose failed\n"); + goto error; + } + + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: write attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write attr - verification failed.\n"); + goto error2; + } + } + + } /* end for */ + + if(H5Tclose(amtype) < 0) { + H5_FAILED(); AT(); + goto error; + } + + return true; + +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + +error2: + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(amtype); + } H5E_END_TRY; + +///dbgf(2, "writer: LEAVE FUNC: write attr - verification failed.\n"); + return false; + +} + +// Temp named pipe works. +#if 0 +static bool +add_attr(state_t *s, hid_t oid,unsigned int which,unsigned num_attrs,const char*aname_fmt,bool sl) { + + //char attrname[sizeof("attr-d-9999999999-999")]; + char attrname[VS_ATTR_NAME_LEN]; + //char* attrname_base= "attr-%u-%u"; + unsigned u; + int i; + unsigned attr_value; + hid_t aid = H5I_INVALID_HID; + hid_t amtype = H5I_INVALID_HID; + hid_t atype = s->filetype; + hid_t sid = s->one_by_one_sid; + + if((amtype = H5Tget_native_type(atype,H5T_DIR_ASCEND)) <0) { + H5_FAILED(); AT(); + printf("H5Tget_native_type failed\n"); + goto error; + } + + /* Add attributes, until just before converting to dense storage */ + for (u = 0; u < num_attrs; u++) { + + /* Create attribute */ + //HDsprintf(attrname, "attr-%u-%u", which,u); + //HDsprintf(attrname, attrname_base, which,u); + HDsprintf(attrname, aname_fmt, which,u); + if((aid = H5Acreate2(oid, attrname, atype, sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Acreate2 failed\n"); + goto error; + } + + attr_value = u+which; + + dbgf(1, "setting attribute %s on group %u to %u\n", attrname, which, u+which); + /* Write data into the attribute */ + if (H5Awrite(aid, amtype, &attr_value) < 0) { + H5_FAILED(); AT(); + printf("H5Awrite failed\n"); + goto error; + } + + /* Close attribute */ + if(H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose failed\n"); + goto error; + } + + if (s->attr_test == true) { + /* Bump up the value of notify to notice the reader to start to read */ + s->np_notify++; + + + dbgf(2, "writer: ready to send the message: %d.\n", s->np_notify); + + if (HDwrite(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) + err(EXIT_FAILURE, "write failed"); + + /* During the wait, writer makes repeated HDF5 API calls + * to trigger EOT at approximately the correct time */ + for(i = 0; i < s->max_lag + 1; i++) { + decisleep(s->tick_len); + H5Aexists(s->file, "nonexistent"); + } + + /* Receive the same value from the reader and verify it before + * going to the next step */ + (s->np_verify)++; + dbgf(2, "writer: ready to receive the message: %d.\n", s->np_verify); + if (HDread(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) + err(EXIT_FAILURE, "read failed"); + + dbgf(2, "writer: finish receiving the message: %d.\n", s->np_notify); + if (s->np_notify != s->np_verify) + errx(EXIT_FAILURE, "received message %d, expecting %d", s->np_notify, s->np_verify); + } + + } /* end for */ + H5Tclose(amtype); +} +#endif + +static bool +modify_attr(state_t *s, hid_t g, const char* aname_fmt,unsigned int which) { + + //char attrname[sizeof("attr-d-9999999999-999")]; + char attrname[VS_ATTR_NAME_LEN]; + hid_t aid = H5I_INVALID_HID; + hid_t amtype = H5I_INVALID_HID; + int modify_value; + + HDsprintf(attrname,aname_fmt,which,0); + if((aid = H5Aopen(g,attrname,H5P_DEFAULT))<0) { + H5_FAILED(); AT(); + printf("H5Aopen failed\n"); + goto error; + } + + if((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND))<0) { + H5_FAILED(); AT(); + printf("H5Tget_native_type failed\n"); + goto error; + } + + // Add later. + //H5T_sign_t h5t_sign = H5Tget_sign(amtype); + /* Unlikely, still make sure -no overflow. */ + if((unsigned int)((int)which)!=which) { + printf("the number of %u causes of overflow when casted to an integer\n",which); + printf("number of iteration is too big, it causes overflow\n"); + goto error; + } + modify_value = (-1)*((int)which); + + if (H5Awrite(aid,amtype,&modify_value) <0) { + H5_FAILED(); AT(); + printf("H5Awrite failed\n"); + goto error; + } + if (H5Tclose(amtype) < 0) { + H5_FAILED(); AT(); + goto error; + } + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + goto error; + } + + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: modify attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write attr - verification failed.\n"); + goto error2; + } + } + + return true; +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(aid); + } H5E_END_TRY; + +error2: + return false; + +} + + +#if 0 +static bool +temp_add_default_group_attr(state_t *s, hid_t g, unsigned int which) { + + const char* aname_format ="attr-%u"; + + add_attr(s,g,which,1,aname_format,false); + +} +#endif + + +static bool +add_default_group_attr(state_t *s, hid_t g, unsigned int which) { + + const char* aname_format ="attr-%u"; + + return add_attr(s,g,which,1,aname_format); + +} + + +static bool +add_vlstr_attr(state_t*s, hid_t g, unsigned int which) { + + hid_t aid = H5I_INVALID_HID; + hid_t atype = H5I_INVALID_HID; + char name[VS_ATTR_NAME_LEN]; + char *astr_val = NULL; + hid_t sid = s->one_by_one_sid; + + astr_val = HDmalloc(VS_ATTR_NAME_LEN); + if (astr_val == NULL) { + H5_FAILED(); AT(); + printf("Allocate memory for buffer failed.\n"); + goto error; + } + + HDsprintf(astr_val,"%u",which); + esnprintf(name, sizeof(name), "attr-%u", which); + + dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); + + /* Create a datatype to refer to. */ + if ((atype = H5Tcopy(H5T_C_S1)) < 0) { + H5_FAILED(); AT(); + printf("Cannot create variable length datatype.\n"); + goto error; + } + + if (H5Tset_size(atype, H5T_VARIABLE) < 0) { + H5_FAILED(); AT(); + printf("Cannot set variable length datatype.\n"); + goto error; + } + + if ((aid = H5Acreate2(g, name, atype, sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Acreate2 failed.\n"); + goto error; + } + + dbgf(1, "astr_val is %s \n", astr_val); + //if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) + //if (H5Awrite(aid, H5T_NATIVE_UINT, astr_val) < 0) + if (H5Awrite(aid, atype, &astr_val) < 0) { + H5_FAILED(); AT(); + printf("H5Awrite failed.\n"); + goto error; + } + + + if (H5Tclose(atype) < 0) { + H5_FAILED(); AT(); + printf("H5Tclose() failed\n"); + goto error; + } + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose() failed\n"); + goto error; + } + + HDfree(astr_val); + + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: write attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write attr - verification failed.\n"); + goto error2; + } + } + + return true; + +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(atype); + } H5E_END_TRY; + + if(astr_val) + HDfree(astr_val); + +error2: + return false; +} + +static bool +del_one_attr(state_t *s, hid_t obj_id,bool is_dense,bool is_vl,unsigned int which) { + + //char attrname[sizeof("attr-d-9999999999-999")]; + char attrname[VS_ATTR_NAME_LEN]; + const char* aname_format_d = "attr-d-%u-%u"; + const char* aname_format = "attr-%u-%u"; + const char* aname_format_vl="attr-%u"; + + + dbgf(2, "writer: coming to delete the attribute.\n"); + if(is_dense == true) + HDsprintf(attrname, aname_format_d, which,0); + else if(is_vl == true) + HDsprintf(attrname, aname_format_vl, which,0); + else + HDsprintf(attrname, aname_format, which,0); + + /* Delete attribute */ + if(H5Adelete(obj_id, attrname) <0) { + H5_FAILED(); AT(); + printf("H5Adelete() failed\n"); + goto error; + } + + if(s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: delete attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: delete attr - verification failed.\n"); + goto error2; + } + } + + return true; + +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + +error2: return false; } + static bool -add_group_attribute(const state_t *s, hid_t g, hid_t sid, unsigned int which) -{ - hid_t aid; - char name[sizeof("attr-9999999999")]; +add_del_vlstr_attr(state_t *s, hid_t g, unsigned int which) { + + bool ret_value = false; + + ret_value = add_vlstr_attr(s,g,which); + if(ret_value == true) + ret_value = del_one_attr(s,g,false,true,which); + + return ret_value; + +} + +static bool +modify_vlstr_attr(state_t*s,hid_t g, unsigned int which) { + + hid_t aid = H5I_INVALID_HID; + hid_t atype = H5I_INVALID_HID; + //char name[sizeof("attr-9999999999")]; + char name[VS_ATTR_NAME_LEN]; + char *astr_val = NULL; + + //astr_val = malloc(sizeof("9999999999!")); + astr_val = HDmalloc(VS_ATTR_NAME_LEN); + if (astr_val == NULL) { + H5_FAILED(); AT(); + printf("Allocate memory for buffer failed.\n"); + goto error; + } + + HDsprintf(astr_val,"%u%c",which,'A'); + //const char *astr_val="test"; + esnprintf(name, sizeof(name), "attr-%u", which); + + dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); + + /* Create a datatype to refer to. */ + if ((atype = H5Tcopy(H5T_C_S1)) < 0) { + H5_FAILED(); AT(); + printf("Cannot create variable length datatype.\n"); + goto error; + } + + if (H5Tset_size(atype, H5T_VARIABLE) < 0) { + H5_FAILED(); AT(); + printf("Cannot set variable length datatype.\n"); + goto error; + } + + //if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, + if ((aid = H5Aopen(g, name, H5P_DEFAULT))<0) { + H5_FAILED(); AT(); + printf("H5Aopen failed.\n"); + goto error; + } + + dbgf(1, "astr_val is %s \n", astr_val); + + //if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) + //if (H5Awrite(aid, H5T_NATIVE_UINT, astr_val) < 0) + if (H5Awrite(aid, atype, &astr_val) < 0) { + H5_FAILED(); AT(); + printf("H5Awrite failed.\n"); + goto error; + } + + if (H5Tclose(atype) < 0) { + H5_FAILED(); AT(); + printf("H5Tclose() failed\n"); + goto error; + } + + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose() failed\n"); + goto error; + } + + HDfree(astr_val); + + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: modify vl attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write attr - verification failed.\n"); + goto error2; + } + } + + return true; + +error: + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(atype); + } H5E_END_TRY; + + if(astr_val) + HDfree(astr_val); + + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + +error2: + return false; + +} + +static bool +add_modify_vlstr_attr(state_t *s, hid_t g, unsigned int which) { + + bool ret_value = false; + //const char* aname_format ="attr-%u"; + ret_value = add_vlstr_attr(s,g,which); + if (true == ret_value) + ret_value = modify_vlstr_attr(s,g,which); + + return ret_value; +} + +static bool +add_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { + + unsigned max_compact = 0; + unsigned min_dense = 0; + const char* aname_format="attr-%u-%u"; + + if(H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense)<0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change() failed\n"); + goto error; + } + + /* Add attributes, until just before converting to dense storage */ + return add_attr(s,g,which,max_compact,aname_format); + + +error: + return false; +} + +static bool +add_attrs_compact_dense(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { + + unsigned max_compact = 0; + unsigned min_dense = 0; + const char* aname_format="attr-d-%u-%u"; + bool ret_value = false; + + if(H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense) < 0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change failed\n"); + goto error; + } + + /* Add attributes, until just before converting to dense storage */ + ret_value = add_attrs_compact(s, g, gcpl, which); + + /* Add another attribute, the storage becomes dense. */ + if(ret_value == true) + ret_value = add_attr(s,g,which+max_compact,1,aname_format); + + return ret_value; + +error: + return false; +} + +#if 0 +static bool +del_one_attr(state_t *s, hid_t obj_id,bool is_dense,unsigned int which) { + + char attrname[sizeof("attr-d-9999999999-999")]; + const char* aname_format_d = "attr-d-%u-%u"; + const char* aname_format = "attr-%u-%u"; + + if(is_dense == true) + HDsprintf(attrname, aname_format_d, which,0); +//printf("attrname is %s\n",attrname); + else + HDsprintf(attrname, aname_format, which,0); + + /* Delete attribute */ + H5Adelete(obj_id, attrname); + nanosleep(&(s->update_interval), NULL); + + +} +#endif + +static bool +del_attrs_compact_dense_compact(state_t *s, hid_t obj_id,hid_t gcpl,unsigned int which) { + + unsigned max_compact = 0; + unsigned min_dense = 0; + unsigned u = 0; + + //char attrname[sizeof("attr-d-9999999999-999")]; + char attrname[VS_ATTR_NAME_LEN]; + const char* aname_format="attr-%u-%u"; + const char* adname_format="attr-d-%u-%u"; + + if (H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense) < 0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change failed\n"); + goto error; + } + u= max_compact +1; +#if 0 + if(max_compact < min_dense) + printf("Minimum number of attributes stored in dense storage should be less than maximum number of attributes stored in compact storage.\n"); +#endif + for(u--;u>=(min_dense-1);u--) { + HDsprintf(attrname, aname_format, which,max_compact-u); + if (H5Adelete(obj_id,attrname) < 0) { + H5_FAILED(); AT(); + printf("H5Adelete failed\n"); + goto error; + } + if(s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: delete attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: delete attr - verification failed.\n"); + goto error2; + } + } + } + + // The writer only deletes the attribute attr-which-0 + HDsprintf(attrname,adname_format,max_compact+which,0); + /// CHECK HERE, add H5Adelete() + if (H5Adelete(obj_id,attrname) < 0) { + H5_FAILED(); AT(); + printf("H5Adelete failed\n"); + goto error; + } + if(s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: delete attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: delete attr - verification failed.\n"); + goto error2; + } + } + + + +// May H5Oget_info3 -- obtain the number of attributes. +//Check the number of attributes >=min_dense. +//We may use the internal function +//is_dense = H5O__is_attr_dense_test(dataset) to check if it is dense in the future. +// + return true; + +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + +error2: + return false; +} + + +static bool +add_del_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { + + bool ret_value = false; + ret_value = add_attrs_compact(s, g, gcpl, which); + if(ret_value == true) { + dbgf(2, "writer: before deleting the attribute.\n"); + ret_value = del_one_attr(s,g,false,false,which); + } + + return ret_value; + +} + +static bool +add_del_attrs_compact_dense(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { + + bool ret_value = false; + unsigned max_compact = 0; + unsigned min_dense = 0; + + if( H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense) < 0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change failed\n"); + goto error; + } + + ret_value = add_attrs_compact_dense(s,g,gcpl,which); + if(ret_value == true) + ret_value = del_one_attr(s,g,true,false,which+max_compact); + + return ret_value; + +error: + return false; + +} + +static bool +add_del_attrs_compact_dense_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { + + bool ret_value = false; + ret_value = add_attrs_compact_dense(s,g,gcpl,which); + if(ret_value == true) + ret_value = del_attrs_compact_dense_compact(s,g,gcpl,which); + + + return ret_value; +} + +static bool +add_modify_default_group_attr(state_t *s, hid_t g, unsigned int which) { + + bool ret_value = false; + const char* aname_format ="attr-%u"; + ret_value = add_default_group_attr(s,g,which); + if(ret_value == true) + ret_value = modify_attr(s,g,aname_format,which); + return ret_value; + +} + +static bool +add_group_attribute(state_t *s, hid_t g, hid_t gcpl, unsigned int which) +{ + + bool ret_value = false; + char test_pattern = s->at_pattern; + + switch (test_pattern) { + case 'c': + ret_value = add_attrs_compact(s, g, gcpl, which); + break; + case 't': + ret_value = add_attrs_compact_dense(s, g, gcpl, which); + break; + case 'C': + ret_value = add_del_attrs_compact(s, g, gcpl, which); + break; + case 'D': + ret_value = add_del_attrs_compact_dense(s, g, gcpl, which); + break; + case 'T': + ret_value = add_del_attrs_compact_dense_compact(s, g, gcpl, which); + break; + case 'M': + ret_value = add_modify_default_group_attr(s, g, which); + break; + case 'v': + ret_value = add_vlstr_attr(s,g, which); + break; + case 'r': + ret_value = add_del_vlstr_attr(s, g, which); + break; + case 'm': + ret_value = add_modify_vlstr_attr(s,g, which); + break; + case 'd': + case ' ': + default: + ret_value = add_default_group_attr(s, g, which); + break; + } +//printf("add_group_attribute return value %d\n",(int)ret_value); + return ret_value; + +} + +#if 0 +{ + hid_t aid; + char name[sizeof("attr-9999999999")]; + + esnprintf(name, sizeof(name), "attr-%u", which); + + dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); + + if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) + errx(EXIT_FAILURE, "H5Acreate2 failed"); + + if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) + errx(EXIT_FAILURE, "H5Awrite failed"); + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); +} +#endif + +static bool +write_group(state_t *s, unsigned int which) +{ + char name[sizeof("/group-9999999999")]; + hid_t g = H5I_INVALID_HID; + hid_t gcpl = H5I_INVALID_HID; + bool result = true; + + //assert(which < s->nsteps); + if (which >= s->nsteps) { + H5_FAILED(); AT(); + printf("Number of created groups is out of bounds\n"); + goto error; + } + + esnprintf(name, sizeof(name), "/group-%d", which); + + gcpl = H5Pcreate(H5P_GROUP_CREATE); + if(gcpl <0) { + H5_FAILED(); AT(); + printf("H5Pcreate failed\n"); + goto error; + } + + /* If we test the dense storage, change the attribute phase. */ + if(s->at_pattern =='d') { + if(H5Pset_attr_phase_change(gcpl, 0, 0) <0) { + H5_FAILED(); AT(); + printf("H5Pset_attr_phase_change failed for the dense storage.\n"); + goto error; + } + } + + if ((g = H5Gcreate2(s->file, name, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Gcreate2 failed\n"); + //np_send_error(s,true); + goto error; + } + + /* If an attribute test is turned on, the NP writer sends a message */ + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + goto error2; + } + } +#if 0 +dbgf(1,"Writer: pass group creation\n"); +#endif + if (s->asteps != 0 && which % s->asteps == 0) + result = add_group_attribute(s, g, gcpl,which); +#if 0 +if(result == true) +printf("Group: successfully receiving the verification from the reader.\n"); +else +printf("Group: Fail to receive the verficiation from the reader.\n"); +#endif + if (H5Gclose(g) < 0) { + H5_FAILED(); AT(); + printf("H5Gclose failed\n"); + goto error; + } + + if(H5Pclose(gcpl) <0) { + H5_FAILED(); AT(); + printf("H5Pclose failed\n"); + goto error; + } + + return result; + +error: + // Consistent + // But if receiving an error message,no need to send the error again. + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + +error2: + + H5E_BEGIN_TRY { + H5Gclose(g); + H5Pclose(gcpl); + } H5E_END_TRY; + + return false; + +} + +// Temp name pipe works. +#if 0 +static bool +write_group(state_t *s, unsigned int which) +{ + char name[sizeof("/group-9999999999")]; + hid_t g; + hid_t gcpl; + int i; + + assert(which < s->nsteps); + + esnprintf(name, sizeof(name), "/group-%d", which); + gcpl = H5Pcreate(H5P_GROUP_CREATE); + if(gcpl <0) + errx(EXIT_FAILURE, "H5Pcreate failed"); + if(s->at_pattern =='d') + H5Pset_attr_phase_change(gcpl, 0, 0); + + g = H5Gcreate2(s->file, name, H5P_DEFAULT, gcpl, H5P_DEFAULT); + // TODO: before the failure, needs to check if the reader is waiting for the pipe. + if (g < 0) + errx(EXIT_FAILURE, "H5Gcreate(, \"%s\", ) failed", name); + if (s->attr_test == true) { + /* Bump up the value of notify to notice the reader to start to read */ + s->np_notify++; + + + dbgf(2, "writer: ready to send the message: %d.\n", s->np_notify); + + if (HDwrite(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) + err(EXIT_FAILURE, "write failed"); + + /* During the wait, writer makes repeated HDF5 API calls + * to trigger EOT at approximately the correct time */ + for(i = 0; i < s->max_lag + 1; i++) { + decisleep(s->tick_len); + H5Aexists(s->file, "nonexistent"); + } + + /* Receive the same value from the reader and verify it before + * going to the next step */ + (s->np_verify)++; + dbgf(2, "writer: ready to receive the message: %d.\n", s->np_verify); + if (HDread(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) + err(EXIT_FAILURE, "read failed"); + + dbgf(2, "writer: finish receiving the message: %d.\n", s->np_notify); + if (s->np_notify != s->np_verify) + errx(EXIT_FAILURE, "received message %d, expecting %d", s->np_notify, s->np_verify); + } + + if (s->asteps != 0 && which % s->asteps == 0) { + add_group_attribute(s, g, gcpl,which); + } + + + if(H5Pclose(gcpl) <0) + errx(EXIT_FAILURE, "H5Pcreate failed"); + + if (H5Gclose(g) < 0) + errx(EXIT_FAILURE, "H5Gclose failed"); + +} + +#endif + + +static bool +vrfy_attr(state_t *s, hid_t g, unsigned int which, char* aname) { + + unsigned int read_which; + hid_t aid = H5I_INVALID_HID; + hid_t amtype = H5I_INVALID_HID; + + //char name[sizeof("attr-d-9999999999-999")]; + + //esnprintf(name, sizeof(name), "attr-%u", which); + //esnprintf(name, sizeof(name), aname_fmt, which); + + if(s->use_named_pipes && true == s->attr_test) { + if(false == np_rd_receive(s)) { + H5_FAILED(); AT(); + goto error2; + } + decisleep(s->tick_len * s->update_interval); + dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); + } + + dbgf(1, "verifying attribute %s on group %u equals %u\n", aname, which, + which); + + +#if 0 + if (H5Sget_simple_extent_npoints(s->one_by_one_sid)!=1) { + dbgf(1, "The number of elements of %s on group %u should be 1, exit.\n" + name,which); + restore_estack(es); + return false; + } +#endif + + if ((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND)) <0) { + H5_FAILED(); AT(); + printf("H5Tget_native_type failed\n"); + goto error; + } + + if ((aid = H5Aopen(g, aname, H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Aopen failed\n"); + goto error; + } + + if (H5Aread(aid, amtype, &read_which) < 0) { + H5_FAILED(); AT(); + printf("H5Aread failed\n"); + goto error; + } + + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose failed\n"); + goto error; + } + + if(read_which != which) { + H5_FAILED(); AT(); + dbgf(2, "reader: the add_attribute verfication failed,expected value is %d\n", which); + dbgf(2, "reader: the add_attribute verfication failed, the value is %d\n", read_which); + printf("The add_attribute verification failed\n"); + goto error; + } + + if(s->use_named_pipes && s->attr_test == true) { + if(np_rd_send(s)==false) + goto error; + dbgf(2, "reader: finish sending back the message: %d\n", s->np_notify); + } + return true; + +error: + H5E_BEGIN_TRY { + H5Tclose(amtype); + H5Aclose(aid); + } H5E_END_TRY; + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,false); +error2: + return false; + +} + +// Temp name pipe works +#if 0 +static bool +vrfy_attr(state_t *s, hid_t g, unsigned int which, char* aname, bool sl) { + + estack_state_t es; + unsigned int read_which; + hid_t aid; + hid_t amtype; + bool ret_value = true; + //char name[sizeof("attr-d-9999999999-999")]; + + //esnprintf(name, sizeof(name), "attr-%u", which); + //esnprintf(name, sizeof(name), aname_fmt, which); + dbgf(1, "verifying attribute %s on group %u equals %u\n", aname, which, + which); + + if(true == s->attr_test) { + s->np_verify++; + /* Receive the notify that the writer bumped up the value */ + if (HDread(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) + err(EXIT_FAILURE, "read failed"); + if (s->np_notify != s->np_verify) + errx(EXIT_FAILURE, "received message %d, expecting %d", s->np_notify, s->np_verify); + decisleep(3*(s->tick_len)); + dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); + + } + + es = disable_estack(); + +#if 0 + if (H5Sget_simple_extent_npoints(s->one_by_one_sid)!=1) { + dbgf(1, "The number of elements of %s on group %u should be 1, exit.\n" + name,which); + restore_estack(es); + return false; + } +#endif + + amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND); + if ((aid = H5Aopen(g, aname, H5P_DEFAULT)) < 0) { + restore_estack(es); + ret_value = false; + } + + if(ret_value == true) { + if (H5Aread(aid, amtype, &read_which) < 0) { + restore_estack(es); + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); + ret_value = false; + } + } + + restore_estack(es); + + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); + + if(ret_value == true) + ret_value = (read_which == which); + + if(s->attr_test == true) { + if(ret_value == false) { + dbgf(2, "reader: the add_attribute verfication failed %d\n", which); + dbgf(2, "reader: the add_attribute verfication failed, the value is %d\n", read_which); + + s->np_notify = 0; + } + if (HDwrite(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) + err(EXIT_FAILURE, "write failed"); + + dbgf(2, "reader: finish sending back the message: %d\n.", s->np_notify); + } + + return ret_value; + +} +#endif + +static bool +verify_default_group_attr(state_t*s,hid_t g, unsigned int which) +{ + //char attrname[sizeof("attr-9999999999")]; + char attrname[VS_ATTR_NAME_LEN]; + const char* aname_format = "attr-%u"; + //bool ret_value = false; + HDsprintf(attrname, aname_format, which); + return vrfy_attr(s,g,which,attrname); + +} + + +static bool +verify_modify_attr(state_t *s, hid_t g, unsigned int which) { + + bool ret = false; + const char* aname_fmt ="attr-%u"; + estack_state_t es; + int read_which; + hid_t aid = H5I_INVALID_HID; + hid_t amtype = H5I_INVALID_HID; + //char aname[sizeof("attr-d-9999999999-999")]; + char attrname[VS_ATTR_NAME_LEN]; + + ret = verify_default_group_attr(s,g,which); + + if(ret == true) { + if(s->use_named_pipes && true == s->attr_test) { + if(false == np_rd_receive(s)) { + H5_FAILED(); AT(); + goto error2; + } + decisleep(s->tick_len * s->update_interval); + dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); + } + + esnprintf(attrname, sizeof(attrname), aname_fmt, which); + //dbgf(1, "verifying attribute %s on group %u equals %u\n", aname, which, + // which); + +#if 0 + if (H5Sget_simple_extent_npoints(s->one_by_one_sid)!=1) { + dbgf(1, "The number of elements of %s on group %u should be 1, exit.\n" + name,which); + restore_estack(es); + return false; + } +#endif + + if ((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND)) < 0) { + H5_FAILED(); AT(); + printf("H5Tget_native_type failed\n"); + goto error; + } + + if ((aid = H5Aopen(g, attrname, H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Aopen failed\n"); + goto error; + } + + if (H5Aread(aid, amtype, &read_which) < 0) { + H5_FAILED(); AT(); + printf("H5Aread failed\n"); + goto error; + } + + if(H5Tclose(amtype) <0) { + H5_FAILED(); AT(); + printf("H5Tclose failed.\n"); + goto error; + } + + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose failed\n"); + goto error; + } + + if((unsigned int)((int)which)!=which) { + H5_FAILED(); AT(); + printf("the unsigned %u causes overflow when casted to signed.\n",which); + printf("number of iteration is too big, it causes overflow.\n"); + goto error; + } + + if(read_which != (-1)*(int)which) { + H5_FAILED(); AT(); + dbgf(2, "reader: the modified_attr() expected value is %d\n", (-1)*(int)which); + dbgf(2, "reader: the modified_attr() actual value is %d\n", read_which); + printf("The modify_attribute verification failed.\n"); + goto error; + } + + if(s->use_named_pipes && s->attr_test == true) { + if(np_rd_send(s)==false) + goto error2; + dbgf(2, "reader: modify_attr finish sending back the message: %d\n", s->np_notify); + } + return true; + + } + return false; + +error: + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(amtype); + } H5E_END_TRY; + + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,false); + +error2: + + return false; +} + +#if 0 +static bool +verify_default_group_attr(state_t*s,hid_t g, unsigned int which) +{ + char attrname[sizeof("attr-9999999999")]; + const char* aname_format = "attr-%u"; + HDsprintf(attrname, aname_format, which); + return vrfy_attr(s,g,which,attrname,false); +} +#endif + +static bool +verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) +{ + estack_state_t es; + //unsigned int read_which; + bool ret = false; + hid_t aid = H5I_INVALID_HID; + hid_t atype = H5I_INVALID_HID; + //char name[sizeof("attr-9999999999")]; + char name[VS_ATTR_NAME_LEN]; + + char *astr_val_exp = NULL; + char * astr_val = NULL; + + if(s->use_named_pipes && true == s->attr_test) { + if(false == np_rd_receive(s)) { + H5_FAILED(); AT(); + goto error2; + } + decisleep(s->tick_len * s->update_interval); + dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); + } + +#if 0 + astr_val = malloc(VS_ATTR_NAME_LEN); + if (astr_val == NULL) { + H5_FAILED(); AT(); + printf("Allocate memory for buffer failed.\n"); + goto error; + } +#endif + + astr_val_exp = HDmalloc(VS_ATTR_NAME_LEN); + if (astr_val_exp == NULL) { + H5_FAILED(); AT(); + printf("Allocate memory for expected buffer failed.\n"); + goto error; + } + + esnprintf(name, sizeof(name), "attr-%u", which); + if(vrfy_mod == true) + HDsprintf(astr_val_exp,"%u%c",which,'A'); + else + HDsprintf(astr_val_exp,"%u",which); + + dbgf(1, "verifying attribute %s on group %u equals %u\n", name, which, + which); + + dbgf(1,"expected vl attr is= %s\n",astr_val_exp); + + es = disable_estack(); + if ((aid = H5Aopen(g, name, H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Aopen failed\n"); + goto error; + } + + /* Create a datatype to refer to. */ + if ((atype = H5Tcopy(H5T_C_S1)) < 0) { + H5_FAILED(); AT(); + printf("Cannot create variable length datatype.\n"); + goto error; + } + + if (H5Tset_size(atype, H5T_VARIABLE) < 0) { + H5_FAILED(); AT(); + printf("Cannot set variable length datatype.\n"); + goto error; + } + + if (H5Aread(aid, atype, &astr_val) < 0) { + H5_FAILED(); AT(); + printf("Cannot read the attribute.\n"); + goto error; + } + + dbgf(1,"read attr is= %s\n",astr_val); + if (HDstrcmp(astr_val, astr_val_exp) != 0) { + H5_FAILED(); AT(); + dbgf(2, "reader: the vl add_attribute verfication failed,expected value is %s\n", astr_val_exp); + dbgf(2, "reader: the vl add_attribute verfication failed, the value is %s\n", astr_val); + printf("The vl add_attribute verification failed\n"); + goto error; + } + + //restore_estack(es); + + if(H5Tclose(atype) <0) { + H5_FAILED(); AT(); + printf("H5Tclose failed.\n"); + goto error; + } + + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose failed.\n"); + goto error; + } + + H5free_memory(astr_val); + HDfree(astr_val_exp); + + if(s->use_named_pipes && s->attr_test == true) { + if(np_rd_send(s)==false) + goto error2; + dbgf(2, "reader: finish sending back the message: %d\n", s->np_notify); + } + + return true; + +error: + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(atype); + } H5E_END_TRY; + if(astr_val) + H5free_memory(astr_val); + if(astr_val_exp) + HDfree(astr_val_exp); + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,false); + +error2: + + return false; + +} + +static bool +verify_del_one_attr(state_t *s,hid_t g, const char *aname) { + + //bool ret = false; + htri_t attr_exists = FALSE; + + if(s->use_named_pipes && true == s->attr_test) { + if(false == np_rd_receive(s)) { + H5_FAILED(); AT(); + goto error2; + } + decisleep(s->tick_len * s->update_interval); + dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); + } + + + attr_exists = H5Aexists_by_name(g,".",aname,H5P_DEFAULT); + if(attr_exists == FALSE) { + dbgf(1,"verify_del_attrs_compact() test: \n"); + dbgf(1," attribute %s is successfully deleted. \n",aname); + } + else if(attr_exists == TRUE) { + dbgf(1,"verify_del_attrs_compact() test failed \n"); + goto error; + } + else{ + dbgf(1,"H5Aexists_by_name failed \n"); + goto error; + } + + if(s->use_named_pipes && s->attr_test == true) { + if(np_rd_send(s)==false) + goto error; + dbgf(2, "reader: finish sending back the message: %d\n", s->np_notify); + } + + return true; +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,false); + +error2: + return false; +} + +static bool +verify_remove_vlstr_attr(state_t* s,hid_t g, unsigned int which) +{ + estack_state_t es; + bool ret = false; + htri_t attr_exists = FALSE; + //char attrname[sizeof("attr-9999999999")]; + char attrname[VS_ATTR_NAME_LEN]; + const char* aname_format = "attr-%u"; + + ret = verify_group_vlstr_attr(s,g,which,false); + if(ret == true) { + HDsprintf(attrname,aname_format,which); + // Add error handling later. + ret = verify_del_one_attr(s,g,attrname); +#if 0 + es = disable_estack(); + attr_exists = H5Aexists_by_name(g,".",attrname,H5P_DEFAULT); + restore_estack(es); + + if(attr_exists == FALSE) { + dbgf(1,"verify_remove_vlstr_attr test: \n"); + dbgf(1," attribute %s on group %u is successfully deleted. \n",attrname,which); + ret = true; + } + else if(attr_exists == TRUE) { + dbgf(1,"verify_remove_vlstr_attr test failed \n"); + ret = false; + } + else{ + dbgf(1,"H5Aexists_by_name failed \n"); + ret = false; + } + +#endif + } + return ret; +} + +static bool +verify_modify_vlstr_attr(state_t *s, hid_t g, unsigned int which){ + + bool ret = false; + + // May change the sid with state_t s + ret = verify_group_vlstr_attr(s,g,which,false); + if(ret == true) + ret = verify_group_vlstr_attr(s,g,which,true); + return ret; + +} + +static bool +verify_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { + + unsigned u; + bool ret = true; + const char* aname_format = "attr-%u-%u"; + //char attrname[sizeof("attr-9999999999-999")]; + char attrname[VS_ATTR_NAME_LEN]; + for (u = 0; u < max_c; u++) { + + HDsprintf(attrname, aname_format, which,u); + if(false == vrfy_attr(s,g,u+which,attrname)) { + ret = false; + break; + } + + } + return ret; + +} + +static bool +verify_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int which) { + + const char* aname_format = "attr-d-%u-%u"; + //char attrname[sizeof("attr-d-9999999999-999")]; + char attrname[VS_ATTR_NAME_LEN]; + bool ret = verify_attrs_compact(s,g,max_c,which); + if(ret == true) { + //HDsprintf(attrname, aname_format, which,0); + HDsprintf(attrname, aname_format, max_c+which,0); + ret = vrfy_attr(s,g,which+max_c,attrname); + if(ret == false) + dbgf(1,"verify_attrs_compact_dense failed \n"); + } + return ret; +} + +static bool +verify_del_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { + + estack_state_t es; + htri_t attr_exists = FALSE; + const char* aname_format = "attr-%u-%u"; + //char attrname[sizeof("attr-d-9999999999-999")]; + char attrname[VS_ATTR_NAME_LEN]; + bool ret = verify_attrs_compact(s,g,max_c,which); + if(ret == true) { + // The writer only deletes the attribute attr-which-0 + HDsprintf(attrname,aname_format,which,0); + // Add error handling later. + //es = disable_estack(); + ret = verify_del_one_attr(s,g,attrname); + } +#if 0 + attr_exists = H5Aexists_by_name(g,".",attrname,H5P_DEFAULT); + restore_estack(es); + if(attr_exists == FALSE) { + dbgf(1,"verify_del_attrs_compact() test: \n"); + dbgf(1," attribute %s on group %u is successfully deleted. \n",attrname,which); + ret = true; + } + else if(attr_exists == TRUE) { + dbgf(1,"verify_del_attrs_compact() test failed \n"); + ret = false; + } + else{ + dbgf(1,"H5Aexists_by_name failed \n"); + ret = false; + } + } +#endif + return ret; +} + +static bool +verify_del_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int which) { + + estack_state_t es; + htri_t attr_exists = FALSE; + const char* aname_format = "attr-d-%u-%u"; + //char attrname[sizeof("attr-d-9999999999-999")]; + char attrname[VS_ATTR_NAME_LEN]; + bool ret = verify_attrs_compact_dense(s,g,max_c,which); + if(ret == true) { + // The writer only deletes the attribute attr-which-0 + HDsprintf(attrname,aname_format,max_c+which,0); + // Add error handling later. + ret = verify_del_one_attr(s,g,attrname); +#if 0 + es = disable_estack(); + attr_exists = H5Aexists_by_name(g,".",attrname,H5P_DEFAULT); + restore_estack(es); + if(attr_exists == FALSE) { + dbgf(1,"verify_del_attrs_compact_dense() test: \n"); + dbgf(1," attribute %s on group %u is successfully deleted. \n",attrname,which); + ret = true; + } + else if(attr_exists == TRUE) { + dbgf(1,"verify_del_attrs_compact_dense() test failed \n"); + ret = false; + } + else{ + dbgf(1,"H5Aexists_by_name failed \n"); + ret = false; + } +#endif + } + return ret; + +} +static bool +verify_del_attrs_compact_dense_compact(state_t *s, + hid_t g, + unsigned max_c, + unsigned min_d, + unsigned int which) { + estack_state_t es; + unsigned u; + htri_t attr_exists = FALSE; + const char* aname_format = "attr-%u-%u"; + char attrname[VS_ATTR_NAME_LEN]; + //char attrname[sizeof("attr-9999999999-999")]; + bool ret = verify_attrs_compact_dense(s,g,max_c,which); + if(ret == true) { + u = max_c + 1; + for(u--;u>=(min_d-1);u--) { + HDsprintf(attrname, aname_format, which,max_c-u); + ret = verify_del_one_attr(s,g,attrname); +#if 0 + es = disable_estack(); + attr_exists = H5Aexists_by_name(g,".",attrname,H5P_DEFAULT); + restore_estack(es); + if(attr_exists == FALSE) { + dbgf(1,"verify_del_attrs_compact_dense_compact() test: \n"); + dbgf(1," attribute %s on group %u is successfully deleted. \n",attrname,which); + ret = true; + } + else if(attr_exists == TRUE) { + dbgf(1,"verify_del_attrs_compact_dense_compact() test failed \n"); + ret = false; + } + else{ + dbgf(1,"H5Aexists_by_name failed \n"); + ret = false; + } +#endif + } + + // The writer only deletes the attribute attr-which-0 + HDsprintf(attrname,aname_format,max_c+which,0); + ret = verify_del_one_attr(s,g,attrname); + // Add error handling later. + // +#if 0 + es = disable_estack(); + attr_exists = H5Aexists_by_name(g,".",attrname,H5P_DEFAULT); + restore_estack(es); + if(attr_exists == FALSE) { + dbgf(1,"verify_del_attrs_compact_dense() test: \n"); + dbgf(1," attribute %s on group %u is successfully deleted. \n",attrname,which); + ret = true; + } + else if(attr_exists == TRUE) { + dbgf(1,"verify_del_attrs_compact_dense() test failed \n"); + ret = false; + } + else{ + dbgf(1,"H5Aexists_by_name failed \n"); + ret = false; + } +#endif + } + return ret; + +} + +static bool +verify_group_attribute(state_t *s, hid_t g, unsigned int which) +{ + char test_pattern = s->at_pattern; + bool ret = false; + unsigned max_compact = 0; + unsigned min_dense = 0; + hid_t gcpl = H5I_INVALID_HID; + switch (test_pattern) { + case 'c': + case 't': + case 'C': + case 'D': + case 'T': + if((gcpl = H5Gget_create_plist(g)) < 0) { + H5_FAILED(); AT(); + printf("H5Gget_create_plist failed\n"); + goto error; + } + if (H5Pget_attr_phase_change(gcpl,&max_compact,&min_dense) < 0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change failed\n"); + goto error; + } + if(H5Pclose(gcpl) < 0) { + H5_FAILED(); AT(); + printf("H5Pclose failed\n"); + goto error; + } + break; + case 'v': + case 'd': + case 'M': + case 'm': + case 'r': + case ' ': + default: + break; + } + switch (test_pattern) { + case 'c': + ret = verify_attrs_compact(s, g, max_compact, which); + break; + case 't': + ret = verify_attrs_compact_dense(s, g, max_compact, which); + break; + case 'C': + ret = verify_del_attrs_compact(s, g, max_compact, which); + break; + case 'D': + ret = verify_del_attrs_compact_dense(s, g, max_compact, which); + break; + case 'T': + ret = verify_del_attrs_compact_dense_compact(s, g, max_compact, min_dense, which); + break; + case 'M': + ret = verify_modify_attr(s, g, which); + break; + case 'v': + ret = verify_group_vlstr_attr(s,g, which,false); + break; + case 'r': + ret = verify_remove_vlstr_attr(s,g, which); + break; + case 'm': + ret = verify_modify_vlstr_attr(s,g, which); + break; + case 'd': + case ' ': + default: + ret = verify_default_group_attr(s, g, which); + break; + } + + return ret; + +error: + /* Still to finish the handshaking */ + if(s->use_named_pipes && s->attr_test == true) { + np_rd_receive(s); + np_send_error(s,false); + } + return false; +} + +static bool +verify_group(state_t *s, unsigned int which) +{ + char name[sizeof("/group-9999999999")]; + hid_t g = H5I_INVALID_HID; + bool result = true; + + //assert(which < s->nsteps); + if (which >= s->nsteps) { + H5_FAILED(); AT(); + printf("Number of created groups is out of bounds\n"); + goto error; + } + + esnprintf(name, sizeof(name), "/group-%d", which); + + if(s->use_named_pipes && true == s->attr_test) { + if(false == np_rd_receive(s)) { + H5_FAILED(); AT(); + goto error; + } + decisleep(s->tick_len * s->update_interval); + dbgf(1, "reader: finish reading the message: %d\n",s->np_notify); + + } + + if((g = H5Gopen(s->file, name, H5P_DEFAULT)) <0) { + H5_FAILED(); AT(); + printf("H5Gopen failed\n"); + if(s->use_named_pipes && s->attr_test == true) { + dbgf(1, "Reader: the H5Gopen verfication failed for group %s \n",name); + np_send_error(s,false); + } + goto error; + } + + if(s->use_named_pipes && s->attr_test == true) { + if(np_rd_send(s)==false) + goto error; + dbgf(1, "Reader: finish sending back the message: %d\n",s->np_notify); + + } + if (s->asteps != 0 && which % s->asteps == 0) + result = verify_group_attribute(s, g, which); + else + result = true; + + if (H5Gclose(g) < 0) { + H5_FAILED(); AT(); + printf("H5Gclose failed\n"); + goto error; + } + + return result; +error: + H5E_BEGIN_TRY { + H5Gclose(g); + } H5E_END_TRY; + + return false; + +} + +// Temp Name pipe works. +#if 0 +static bool +verify_group(state_t *s, unsigned int which) +{ + char name[sizeof("/group-9999999999")]; + hid_t g; + estack_state_t es; + bool result = true; + bool gopen_ret = true; + + assert(which < s->nsteps); + + esnprintf(name, sizeof(name), "/group-%d", which); + + if(true == s->attr_test) { + s->np_verify++; + /* Receive the notify that the writer bumped up the value */ + if (HDread(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) + err(EXIT_FAILURE, "read failed"); + if (s->np_notify != s->np_verify) + errx(EXIT_FAILURE, "received message %d, expecting %d", s->np_notify, s->np_verify); + decisleep(3*(s->tick_len)); + dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); + + } + + es = disable_estack(); + g = H5Gopen(s->file, name, H5P_DEFAULT); + restore_estack(es); + + if (g < 0) + gopen_ret = false; +//if(gopen_ret == true) { +if(s->attr_test == true) { + if(gopen_ret == false) { + dbgf(1, "reader: the gopen verfication failed \n",which); + + s->np_notify = 0; + } + if (HDwrite(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) + err(EXIT_FAILURE, "write failed"); + + dbgf(1, "reader: finish sending back the message: %d\n.", s->np_notify); + } + +if(gopen_ret == true) { + if (s->asteps != 0 && which % s->asteps == 0) + result = verify_group_attribute(s, g, which); + else + result = true; + + +//} + + if (H5Gclose(g) < 0) + errx(EXIT_FAILURE, "H5Gclose failed"); +} +else + result = false; + + return result; +} + +#endif + + +#if 0 +static bool +add_group_attribute(state_t *s, hid_t g, hid_t sid, unsigned int which) +{ + hid_t aid; + char name[sizeof("attr-9999999999")]; + + esnprintf(name, sizeof(name), "attr-%u", which); + + dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); + + if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) + errx(EXIT_FAILURE, "H5Acreate2 failed"); + + if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) + errx(EXIT_FAILURE, "H5Awrite failed"); + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); +} + + +static bool +write_group(state_t *s, unsigned int which) +{ + char name[sizeof("/group-9999999999")]; + hid_t g; + + assert(which < s->nsteps); + + esnprintf(name, sizeof(name), "/group-%d", which); + + g = H5Gcreate2(s->file, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + if (g < 0) + errx(EXIT_FAILURE, "H5Gcreate(, \"%s\", ) failed", name); + + if (s->asteps != 0 && which % s->asteps == 0) + add_group_attribute(s, g, s->one_by_one_sid, which); + + if (H5Gclose(g) < 0) + errx(EXIT_FAILURE, "H5Gclose failed"); +} + +static bool +verify_group_attribute(hid_t g, unsigned int which) +{ + estack_state_t es; + unsigned int read_which; + hid_t aid; + char name[sizeof("attr-9999999999")]; + + esnprintf(name, sizeof(name), "attr-%u", which); + + dbgf(1, "verifying attribute %s on group %u equals %u\n", name, which, + which); + + es = disable_estack(); + if ((aid = H5Aopen(g, name, H5P_DEFAULT)) < 0) { + restore_estack(es); + return false; + } + + if (H5Aread(aid, H5T_NATIVE_UINT, &read_which) < 0) { + restore_estack(es); + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); + return false; + } + + restore_estack(es); + + if (H5Aclose(aid) < 0) + errx(EXIT_FAILURE, "H5Aclose failed"); + + return read_which == which; +} + +static bool +verify_group(state_t *s, unsigned int which) +{ + char name[sizeof("/group-9999999999")]; + hid_t g; + estack_state_t es; + bool result; + + assert(which < s->nsteps); + + esnprintf(name, sizeof(name), "/group-%d", which); + + es = disable_estack(); + g = H5Gopen(s->file, name, H5P_DEFAULT); + restore_estack(es); + + if (g < 0) + return false; + + if (s->asteps != 0 && which % s->asteps == 0) + result = verify_group_attribute(g, which); + else + result = true; + + if (H5Gclose(g) < 0) + errx(EXIT_FAILURE, "H5Gclose failed"); + + return result; +} + +#endif + +//OLDWORK +#if 0 +static bool +add_group_attribute(const state_t *s, hid_t g, hid_t sid, unsigned int which) +{ + hid_t aid; + char name[sizeof("attr-9999999999")]; + + esnprintf(name, sizeof(name), "attr-%u", which); + + dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); + + if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Acreate2 failed\n"); + goto error; + } + + if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) { + H5_FAILED(); AT(); + printf("H5Awrite failed\n"); + goto error; + } + + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose failed\n"); + goto error; + } + + return true; + +error: + H5E_BEGIN_TRY { + H5Aclose(aid); + } H5E_END_TRY; + + return false; +} + +static bool +write_group(state_t *s, unsigned int which) +{ + char name[sizeof("/group-9999999999")]; + hid_t g = H5I_INVALID_HID; + bool result = true; + + if (which >= s->nsteps) { + H5_FAILED(); AT(); + printf("group order is out of bounds\n"); + goto error; + } + + esnprintf(name, sizeof(name), "/group-%d", which); + + if ((g = H5Gcreate2(s->file, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Gcreate2 failed\n"); + goto error; + } + + if (s->asteps != 0 && which % s->asteps == 0) + result = add_group_attribute(s, g, s->one_by_one_sid, which); + + if (H5Gclose(g) < 0) { + H5_FAILED(); AT(); + printf("H5Gclose failed\n"); + goto error; + } + + return result; + +error: + H5E_BEGIN_TRY { + H5Gclose(g); + } H5E_END_TRY; + + return false; +} + +static bool +verify_group_attribute(hid_t g, unsigned int which) +{ + unsigned int read_which; + hid_t aid; + char name[sizeof("attr-9999999999")]; + + esnprintf(name, sizeof(name), "attr-%u", which); + + dbgf(1, "verifying attribute %s on group %u equals %u\n", name, which, which); + + if ((aid = H5Aopen(g, name, H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Aopen failed\n"); + goto error; + } + + if (H5Aread(aid, H5T_NATIVE_UINT, &read_which) < 0) { + H5_FAILED(); AT(); + printf("H5Aread failed\n"); + goto error; + } + + if (read_which != which) { + H5_FAILED(); AT(); + printf("H5Aread wrong value\n"); + goto error; + } + + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aread failed\n"); + goto error; + } + + return true; + +error: + H5E_BEGIN_TRY { + H5Aclose(aid); + } H5E_END_TRY; + + return false; +} + +static bool +verify_group(state_t *s, unsigned int which) +{ + char name[sizeof("/group-9999999999")]; + hid_t g = H5I_INVALID_HID; + bool result = true; + + if (which >= s->nsteps) { + H5_FAILED(); AT(); + printf("Group order is out of bounds\n"); + goto error; + } + + esnprintf(name, sizeof(name), "/group-%d", which); + + if ((g = H5Gopen(s->file, name, H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Gopen failed\n"); + goto error; + } + + if (s->asteps != 0 && which % s->asteps == 0) + result = verify_group_attribute(g, which); + else + result = true; + + if (H5Gclose(g) < 0) { + H5_FAILED(); AT(); + printf("H5Gclose failed\n"); + goto error; + } + + return result; + +error: + H5E_BEGIN_TRY { + H5Gclose(g); + } H5E_END_TRY; + + return false; +} + +int +main(int argc, char **argv) +{ + hid_t fapl = H5I_INVALID_HID, fcpl = H5I_INVALID_HID; + unsigned step; + bool writer = false; + state_t s; + const char *personality; + H5F_vfd_swmr_config_t config; + const char *fifo_writer_to_reader = "./fifo_group_writer_to_reader"; + const char *fifo_reader_to_writer = "./fifo_group_reader_to_writer"; + int fd_writer_to_reader = -1, fd_reader_to_writer = -1; + int notify = 0, verify = 0; + unsigned int i; + + if (!state_init(&s, argc, argv)) { + H5_FAILED(); AT(); + printf("state_init failed\n"); + goto error; + } + + personality = strstr(s.progname, "vfd_swmr_group_"); + + if (personality != NULL && strcmp(personality, "vfd_swmr_group_writer") == 0) + writer = true; + else if (personality != NULL && strcmp(personality, "vfd_swmr_group_reader") == 0) + writer = false; + else { + H5_FAILED(); AT(); + printf("unknown personality, expected vfd_swmr_group_{reader,writer}\n"); + goto error; + } + + /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ + init_vfd_swmr_config(&config, 4, 7, writer, FALSE, 128, "./group-shadow"); + + /* use_latest_format, use_vfd_swmr, only_meta_page, config */ + if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config)) < 0) { + H5_FAILED(); AT(); + printf("vfd_swmr_create_fapl failed\n"); + goto error; + } + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) { + H5_FAILED(); AT(); + printf("H5Pcreate failed\n"); + goto error; + } + + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1) < 0) { + H5_FAILED(); AT(); + printf("H5Pset_file_space_strategy failed\n"); + goto error; + } + + if (writer) + s.file = H5Fcreate(s.filename, H5F_ACC_TRUNC, fcpl, fapl); + else + s.file = H5Fopen(s.filename, H5F_ACC_RDONLY, fapl); + + if (s.file < 0) { + H5_FAILED(); AT(); + printf("H5Fcreate/open failed\n"); + goto error; + } + + /* Use two named pipes(FIFO) to coordinate the writer and reader for + * two-way communication so that the two sides can move forward together. + * One is for the writer to write to the reader. + * The other one is for the reader to signal the writer. */ + if (s.use_named_pipes && writer) { + /* Writer creates two named pipes(FIFO) */ + if (HDmkfifo(fifo_writer_to_reader, 0600) < 0) { + H5_FAILED(); AT(); + printf("HDmkfifo failed\n"); + goto error; + } + + if (HDmkfifo(fifo_reader_to_writer, 0600) < 0) { + H5_FAILED(); AT(); + printf("HDmkfifo failed\n"); + goto error; + } + + } + + /* Both the writer and reader open the pipes */ + if (s.use_named_pipes && (fd_writer_to_reader = HDopen(fifo_writer_to_reader, O_RDWR)) < 0) { + H5_FAILED(); AT(); + printf("HDopen failed\n"); + goto error; + } + + if (s.use_named_pipes && (fd_reader_to_writer = HDopen(fifo_reader_to_writer, O_RDWR)) < 0) { + H5_FAILED(); AT(); + printf("HDopen failed\n"); + goto error; + } + + if (writer) { + for (step = 0; step < s.nsteps; step++) { + dbgf(2, "writer: step %d\n", step); + + if (!write_group(&s, step)) { + H5_FAILED(); AT(); + printf("write_group failed\n"); + + /* At communication interval, notifies the reader about the failture and quit */ + if (s.use_named_pipes && (step % s.csteps == 0)) { + notify = -1; + HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)); + } + + goto error; + } else { + /* At communication interval, notifies the reader and waits for its response */ + if (s.use_named_pipes && (step % s.csteps == 0)) { + /* Bump up the value of notify to notice the reader to start to read */ + notify++; + if (HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDwrite failed\n"); + goto error; + } + + /* During the wait, writer makes repeated HDF5 API calls + * to trigger EOT at approximately the correct time */ + for(i = 0; i < config.max_lag + 1; i++) { + decisleep(config.tick_len); + H5E_BEGIN_TRY { + H5Aexists(s.file, "nonexistent"); + } H5E_END_TRY; + } + + /* Receive the same value from the reader and verify it before + * going to the next step */ + verify++; + if (HDread(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDread failed\n"); + goto error; + } + + if (notify == -1) { + H5_FAILED(); AT(); + printf("reader failed to verify group\n"); + goto error; + } + + if (notify != verify) { + H5_FAILED(); AT(); + printf("received message %d, expecting %d\n", notify, verify); + goto error; + } + } + } + } + } + else { + for (step = 0; step < s.nsteps; step++) { + dbgf(2, "reader: step %d\n", step); + + /* At communication interval, waits for the writer to finish creation before starting verification + */ + if (s.use_named_pipes && (step % s.csteps == 0)) { + /* The writer should have bumped up the value of notify. + * Do the same with verify and confirm it */ + verify++; + + /* Receive the notify that the writer bumped up the value */ + if (HDread(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDread failed\n"); + goto error; + } + + if (notify == -1) { + H5_FAILED(); AT(); + printf("writer failed to create group\n"); + goto error; + } + + if (notify != verify) { + H5_FAILED(); AT(); + printf("received message %d, expecting %d\n", notify, verify); + goto error; + } + } + + /* Wait for a few ticks for the update to happen */ + if (s.use_named_pipes) + decisleep(config.tick_len * s.update_interval); + + /* Start to verify group */ + if (!verify_group(&s, step)) { + H5_FAILED(); AT(); + printf("verify_group failed\n"); - esnprintf(name, sizeof(name), "attr-%u", which); + /* At communication interval, tell the writer about the failure and exit */ + if (s.use_named_pipes && (step % s.csteps == 0)) { + notify = -1; + HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)); + } - dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); + goto error; + } else { + if (s.use_named_pipes && (step % s.csteps == 0)) { + /* Send back the same nofity value for acknowledgement to tell the writer + * move to the next step */ + if (HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDwrite failed\n"); + goto error; + } + } + } + } + } - if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, - H5P_DEFAULT)) < 0) { + if (H5Pclose(fapl) < 0) { H5_FAILED(); AT(); - printf("H5Acreate2 failed\n"); + printf("H5Pclose failed\n"); goto error; } - if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) { + if (H5Pclose(fcpl) < 0) { H5_FAILED(); AT(); - printf("H5Awrite failed\n"); + printf("H5Pclose failed\n"); goto error; } - if (H5Aclose(aid) < 0) { + if (H5Fclose(s.file) < 0) { H5_FAILED(); AT(); - printf("H5Aclose failed\n"); + printf("H5Fclose failed\n"); goto error; } - return true; - -error: - H5E_BEGIN_TRY { - H5Aclose(aid); - } H5E_END_TRY; - - return false; -} - -static bool -write_group(state_t *s, unsigned int which) -{ - char name[sizeof("/group-9999999999")]; - hid_t g = H5I_INVALID_HID; - bool result = true; - - if (which >= s->nsteps) { + /* Both the writer and reader close the named pipes */ + if (s.use_named_pipes && HDclose(fd_writer_to_reader) < 0) { H5_FAILED(); AT(); - printf("group order is out of bounds\n"); + printf("HDclose failed\n"); goto error; } - esnprintf(name, sizeof(name), "/group-%d", which); - - if ((g = H5Gcreate2(s->file, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) { + if (s.use_named_pipes && HDclose(fd_reader_to_writer) < 0) { H5_FAILED(); AT(); - printf("H5Gcreate2 failed\n"); + printf("HDclose failed\n"); goto error; } - if (s->asteps != 0 && which % s->asteps == 0) - result = add_group_attribute(s, g, s->one_by_one_sid, which); + /* Reader finishes last and deletes the named pipes */ + if(s.use_named_pipes && !writer) { + if(HDremove(fifo_writer_to_reader) != 0) { + H5_FAILED(); AT(); + printf("HDremove failed\n"); + goto error; + } - if (H5Gclose(g) < 0) { - H5_FAILED(); AT(); - printf("H5Gclose failed\n"); - goto error; + if(HDremove(fifo_reader_to_writer) != 0) { + H5_FAILED(); AT(); + printf("HDremove failed\n"); + goto error; + } } - return result; + return EXIT_SUCCESS; error: H5E_BEGIN_TRY { - H5Gclose(g); + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(s.file); } H5E_END_TRY; - return false; -} + if (s.use_named_pipes && fd_writer_to_reader >= 0) + HDclose(fd_writer_to_reader); -static bool -verify_group_attribute(hid_t g, unsigned int which) -{ - unsigned int read_which; - hid_t aid; - char name[sizeof("attr-9999999999")]; + if (s.use_named_pipes && fd_reader_to_writer >= 0) + HDclose(fd_reader_to_writer); - esnprintf(name, sizeof(name), "attr-%u", which); + if(s.use_named_pipes && !writer) { + HDremove(fifo_writer_to_reader); + HDremove(fifo_reader_to_writer); + } - dbgf(1, "verifying attribute %s on group %u equals %u\n", name, which, which); + return EXIT_FAILURE; +} +#endif - if ((aid = H5Aopen(g, name, H5P_DEFAULT)) < 0) { - H5_FAILED(); AT(); - printf("H5Aopen failed\n"); - goto error; - } +int +main(int argc, char **argv) +{ - if (H5Aread(aid, H5T_NATIVE_UINT, &read_which) < 0) { - H5_FAILED(); AT(); - printf("H5Aread failed\n"); - goto error; - } +#if 0 + hid_t fapl, fcpl; + herr_t ret; + unsigned step; + bool writer; + state_t s; + const char *personality; + H5F_vfd_swmr_config_t config; + const char *fifo_writer_to_reader = "./fifo_group_writer_to_reader"; + const char *fifo_reader_to_writer = "./fifo_group_reader_to_writer"; + int fd_writer_to_reader, fd_reader_to_writer; + // notify = 0 and verify = 0 are for error. + int notify = 1, verify = 1; + unsigned int i; - if (read_which != which) { - H5_FAILED(); AT(); - printf("H5Aread wrong value\n"); - goto error; - } + state_init(&s, argc, argv); - if (H5Aclose(aid) < 0) { - H5_FAILED(); AT(); - printf("H5Aread failed\n"); - goto error; + personality = strstr(s.progname, "vfd_swmr_group_"); + + if (personality != NULL && + strcmp(personality, "vfd_swmr_group_writer") == 0) + writer = true; + else if (personality != NULL && + strcmp(personality, "vfd_swmr_group_reader") == 0) + writer = false; + else { + errx(EXIT_FAILURE, + "unknown personality, expected vfd_swmr_group_{reader,writer}"); } - return true; + /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ + init_vfd_swmr_config(&config, 4, 7, writer, FALSE, 128, "./group-shadow"); -error: - H5E_BEGIN_TRY { - H5Aclose(aid); - } H5E_END_TRY; + /* use_latest_format, use_vfd_swmr, only_meta_page, config */ + fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config); - return false; -} + if (fapl < 0) + errx(EXIT_FAILURE, "vfd_swmr_create_fapl"); -static bool -verify_group(state_t *s, unsigned int which) -{ - char name[sizeof("/group-9999999999")]; - hid_t g = H5I_INVALID_HID; - bool result = true; + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + errx(EXIT_FAILURE, "H5Pcreate"); - if (which >= s->nsteps) { - H5_FAILED(); AT(); - printf("Group order is out of bounds\n"); - goto error; - } + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1); + if (ret < 0) + errx(EXIT_FAILURE, "H5Pset_file_space_strategy"); - esnprintf(name, sizeof(name), "/group-%d", which); + if (writer) + s.file = H5Fcreate(s.filename, H5F_ACC_TRUNC, fcpl, fapl); + else + s.file = H5Fopen(s.filename, H5F_ACC_RDONLY, fapl); - if ((g = H5Gopen(s->file, name, H5P_DEFAULT)) < 0) { - H5_FAILED(); AT(); - printf("H5Gopen failed\n"); - goto error; - } + if (s.file == badhid) + errx(EXIT_FAILURE, writer ? "H5Fcreate" : "H5Fopen"); - if (s->asteps != 0 && which % s->asteps == 0) - result = verify_group_attribute(g, which); - else - result = true; + /* Use two named pipes(FIFO) to coordinate the writer and reader for + * two-way communication so that the two sides can move forward together. + * One is for the writer to write to the reader. + * The other one is for the reader to signal the writer. */ + if (writer) { + /* Writer creates two named pipes(FIFO) */ + if (HDmkfifo(fifo_writer_to_reader, 0600) < 0) + errx(EXIT_FAILURE, "HDmkfifo"); - if (H5Gclose(g) < 0) { - H5_FAILED(); AT(); - printf("H5Gclose failed\n"); - goto error; + if (HDmkfifo(fifo_reader_to_writer, 0600) < 0) + errx(EXIT_FAILURE, "HDmkfifo"); } - return result; + /* Both the writer and reader open the pipes */ + if ((fd_writer_to_reader = HDopen(fifo_writer_to_reader, O_RDWR)) < 0) + errx(EXIT_FAILURE, "fifo_writer_to_reader open failed"); -error: - H5E_BEGIN_TRY { - H5Gclose(g); - } H5E_END_TRY; + if ((fd_reader_to_writer = HDopen(fifo_reader_to_writer, O_RDWR)) < 0) + errx(EXIT_FAILURE, "fifo_reader_to_writer open failed"); - return false; -} +#endif -int -main(int argc, char **argv) -{ hid_t fapl = H5I_INVALID_HID, fcpl = H5I_INVALID_HID; unsigned step; bool writer = false; @@ -355,6 +2858,8 @@ main(int argc, char **argv) int fd_writer_to_reader = -1, fd_reader_to_writer = -1; int notify = 0, verify = 0; unsigned int i; + bool wg_ret = false; + bool vg_ret = false; if (!state_init(&s, argc, argv)) { H5_FAILED(); AT(); @@ -364,9 +2869,11 @@ main(int argc, char **argv) personality = strstr(s.progname, "vfd_swmr_group_"); - if (personality != NULL && strcmp(personality, "vfd_swmr_group_writer") == 0) + if (personality != NULL && + strcmp(personality, "vfd_swmr_group_writer") == 0) writer = true; - else if (personality != NULL && strcmp(personality, "vfd_swmr_group_reader") == 0) + else if (personality != NULL && + strcmp(personality, "vfd_swmr_group_reader") == 0) writer = false; else { H5_FAILED(); AT(); @@ -440,27 +2947,51 @@ main(int argc, char **argv) goto error; } + if(s.use_named_pipes) { + s.np_fd_w_to_r = fd_writer_to_reader; + s.np_fd_r_to_w = fd_reader_to_writer; + s.np_notify = notify; + s.np_verify = verify; + s.tick_len = config.tick_len; + s.max_lag = config.max_lag; + } + + // TODO: use a different name for s pointer(sp?) in subroutine, + // TODO: since its name is also defined as s. + + // For attribute test, force the named pipe to communicate for every step. + if (s.at_pattern != ' ') { + s.attr_test = true; + if(s.use_named_pipes) + s.csteps = 1; + } if (writer) { for (step = 0; step < s.nsteps; step++) { dbgf(2, "writer: step %d\n", step); - if (!write_group(&s, step)) { + wg_ret = write_group(&s, step); + + if(wg_ret == false) { H5_FAILED(); AT(); - printf("write_group failed\n"); + printf("write_group failed at step %d\n",step); + /* At communication interval, notifies the reader about the failture and quit */ - if (s.use_named_pipes && (step % s.csteps == 0)) { - notify = -1; - HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)); + if (s.use_named_pipes && s.attr_test !=true && step % s.csteps == 0) { + //if(1){ + s.np_notify = -1; + HDwrite(fd_writer_to_reader, &(s.np_notify), sizeof(int)); } - goto error; - } else { + } + else { + /* At communication interval, notifies the reader and waits for its response */ - if (s.use_named_pipes && (step % s.csteps == 0)) { + if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { + /* Bump up the value of notify to notice the reader to start to read */ - notify++; - if (HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { + s.np_notify++; + if (HDwrite(fd_writer_to_reader, &(s.np_notify), sizeof(int)) < 0) { H5_FAILED(); AT(); printf("HDwrite failed\n"); goto error; @@ -477,86 +3008,91 @@ main(int argc, char **argv) /* Receive the same value from the reader and verify it before * going to the next step */ - verify++; - if (HDread(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { + (s.np_verify)++; + if (HDread(fd_reader_to_writer, &(s.np_notify), sizeof(int)) < 0){ H5_FAILED(); AT(); printf("HDread failed\n"); goto error; } - if (notify == -1) { + if (s.np_notify == -1) { H5_FAILED(); AT(); printf("reader failed to verify group\n"); goto error; } - if (notify != verify) { + if (s.np_notify != s.np_verify) { H5_FAILED(); AT(); - printf("received message %d, expecting %d\n", notify, verify); + printf("received message %d, expecting %d\n", s.np_notify, s.np_verify); goto error; } } } } - } - else { - for (step = 0; step < s.nsteps; step++) { - dbgf(2, "reader: step %d\n", step); + } else { + for (step = 0; step < s.nsteps;step++) { + dbgf(1, "reader: step %d\n", step); - /* At communication interval, waits for the writer to finish creation before starting verification - */ - if (s.use_named_pipes && (step % s.csteps == 0)) { + /* At communication interval, waits for the writer to finish creation before starting verification */ + if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { /* The writer should have bumped up the value of notify. * Do the same with verify and confirm it */ - verify++; + s.np_verify++; /* Receive the notify that the writer bumped up the value */ - if (HDread(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { + if (HDread(fd_writer_to_reader, &(s.np_notify), sizeof(int)) < 0) { H5_FAILED(); AT(); printf("HDread failed\n"); goto error; } - if (notify == -1) { + if (s.np_notify == -1) { H5_FAILED(); AT(); printf("writer failed to create group\n"); goto error; } - if (notify != verify) { + if (s.np_notify != s.np_verify) { H5_FAILED(); AT(); - printf("received message %d, expecting %d\n", notify, verify); + printf("received message %d, expecting %d\n", s.np_notify, s.np_verify); goto error; } } - /* Wait for a few ticks for the update to happen */ - if (s.use_named_pipes) + /* For the default test, wait for a few ticks for the update to happen */ + if(s.use_named_pipes && s.attr_test== false) decisleep(config.tick_len * s.update_interval); - /* Start to verify group */ - if (!verify_group(&s, step)) { - H5_FAILED(); AT(); - printf("verify_group failed\n"); + vg_ret = verify_group(&s, step); + + if (vg_ret == false) { + printf("verify_group failed\n"); + H5_FAILED(); AT(); /* At communication interval, tell the writer about the failure and exit */ - if (s.use_named_pipes && (step % s.csteps == 0)) { - notify = -1; - HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)); + if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { + //if(1){ + s.np_notify = -1; + HDwrite(fd_reader_to_writer, &(s.np_notify), sizeof(int)); } - goto error; - } else { - if (s.use_named_pipes && (step % s.csteps == 0)) { - /* Send back the same nofity value for acknowledgement to tell the writer - * move to the next step */ - if (HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { + + } + else { + + /* Send back the same nofity value for acknowledgement to tell the writer + * move to the next step. */ + // TO THINK: reader will never have a chance to acknowledge the writer when attribute verfication occurs. + // RESOLVED, no need to carry out the following for the attribute operation. It is done in the attribute level. + if (s.use_named_pipes && s.attr_test!=true && step % s.csteps == 0) { + if (HDwrite(fd_reader_to_writer, &(s.np_notify), sizeof(int)) < 0) { H5_FAILED(); AT(); printf("HDwrite failed\n"); goto error; } } } + } } @@ -627,6 +3163,7 @@ main(int argc, char **argv) } return EXIT_FAILURE; + } #else /* H5_HAVE_WIN32_API */ From 679a529f966fa81054ed05753232f511751c0ab2 Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Wed, 21 Apr 2021 12:52:34 -0500 Subject: [PATCH 02/13] Make the modifying attribute value obvious --- test/vfd_swmr_group_writer.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index b695b8335db..3fd49b28378 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -531,6 +531,7 @@ modify_attr(state_t *s, hid_t g, const char* aname_fmt,unsigned int which) { goto error; } +#if 0 // Add later. //H5T_sign_t h5t_sign = H5Tget_sign(amtype); /* Unlikely, still make sure -no overflow. */ @@ -539,7 +540,8 @@ modify_attr(state_t *s, hid_t g, const char* aname_fmt,unsigned int which) { printf("number of iteration is too big, it causes overflow\n"); goto error; } - modify_value = (-1)*((int)which); +#endif + modify_value = which+10000; if (H5Awrite(aid,amtype,&modify_value) <0) { H5_FAILED(); AT(); @@ -1533,14 +1535,16 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { goto error; } +#if 0 if((unsigned int)((int)which)!=which) { H5_FAILED(); AT(); printf("the unsigned %u causes overflow when casted to signed.\n",which); printf("number of iteration is too big, it causes overflow.\n"); goto error; } +#endif - if(read_which != (-1)*(int)which) { + if(read_which != (which+10000)) { H5_FAILED(); AT(); dbgf(2, "reader: the modified_attr() expected value is %d\n", (-1)*(int)which); dbgf(2, "reader: the modified_attr() actual value is %d\n", read_which); @@ -2974,13 +2978,15 @@ main(int argc, char **argv) if(wg_ret == false) { H5_FAILED(); AT(); printf("write_group failed at step %d\n",step); - /* At communication interval, notifies the reader about the failture and quit */ if (s.use_named_pipes && s.attr_test !=true && step % s.csteps == 0) { //if(1){ + #if 0 s.np_notify = -1; HDwrite(fd_writer_to_reader, &(s.np_notify), sizeof(int)); + #endif + np_send_error(&s,true); } goto error; } @@ -2989,6 +2995,12 @@ main(int argc, char **argv) /* At communication interval, notifies the reader and waits for its response */ if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { + if(np_wr_send_receive(&s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write group - verification failed.\n"); + goto error; + } +#if 0 /* Bump up the value of notify to notice the reader to start to read */ s.np_notify++; if (HDwrite(fd_writer_to_reader, &(s.np_notify), sizeof(int)) < 0) { @@ -3026,6 +3038,7 @@ main(int argc, char **argv) printf("received message %d, expecting %d\n", s.np_notify, s.np_verify); goto error; } +#endif } } } @@ -3035,6 +3048,11 @@ main(int argc, char **argv) /* At communication interval, waits for the writer to finish creation before starting verification */ if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { + if(false == np_rd_receive(&s)) { + H5_FAILED(); AT(); + goto error; + } +#if 0 /* The writer should have bumped up the value of notify. * Do the same with verify and confirm it */ s.np_verify++; @@ -3057,6 +3075,7 @@ main(int argc, char **argv) printf("received message %d, expecting %d\n", s.np_notify, s.np_verify); goto error; } +#endif } /* For the default test, wait for a few ticks for the update to happen */ @@ -3072,8 +3091,9 @@ main(int argc, char **argv) /* At communication interval, tell the writer about the failure and exit */ if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { //if(1){ - s.np_notify = -1; - HDwrite(fd_reader_to_writer, &(s.np_notify), sizeof(int)); + np_send_error(&s,false); + //s.np_notify = -1; + //HDwrite(fd_reader_to_writer, &(s.np_notify), sizeof(int)); } goto error; @@ -3085,11 +3105,15 @@ main(int argc, char **argv) // TO THINK: reader will never have a chance to acknowledge the writer when attribute verfication occurs. // RESOLVED, no need to carry out the following for the attribute operation. It is done in the attribute level. if (s.use_named_pipes && s.attr_test!=true && step % s.csteps == 0) { +#if 0 if (HDwrite(fd_reader_to_writer, &(s.np_notify), sizeof(int)) < 0) { H5_FAILED(); AT(); printf("HDwrite failed\n"); goto error; } +#endif + if(np_rd_send(&s)==false) + goto error; } } From a0c9f955f0c664df74d08ba0410393e3894d74a1 Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Wed, 21 Apr 2021 16:23:23 -0500 Subject: [PATCH 03/13] Correct the debugging mismatch issue, add more comments. --- test/vfd_swmr_group_writer.c | 159 +++++++++++++++++++++++++++-------- 1 file changed, 122 insertions(+), 37 deletions(-) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 3fd49b28378..b0957e49019 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -68,15 +68,37 @@ static void usage(const char *progname) { fprintf(stderr, "usage: %s [-S] [-a steps] [-b] [-c]\n" - " [-n iterations] [-N] [-q] [-u numb_ticks]\n" + " [-n iterations] [-N] [-q] [-u numb_ticks] [-A at_pattern]\n" "\n" "-S: do not use VFD SWMR\n" "-a steps: `steps` between adding attributes\n" "-b: write data in big-endian byte order\n" "-c steps: `steps` between communication between the writer and reader\n" "-n ngroups: the number of groups\n" - "-N: do not use named pipes, mainly for running the writer and reader seperately\n" - "-u numb_tcks: `numb_ticks` for the reader to wait before verification\n" + "-N: do not use named pipes, \n" + " mainly for running the writer and reader seperately\n" + "-u numb_ticks: `numb_ticks` for the reader to wait before verification\n" + "-A at_pattern: `at_pattern' for different attribute tests\n" + " The value of `at_pattern` is one of the following:\n" + " `compact` - Attributes added in compact storage\n" + " `dense` - An attribute added in dense storage\n" + " `compact-to-dense` - Attributes added first in compact\n" + " then in dense storage\n" + " `compact-del` - Attributes added and then one\n" + " attribute deleted, in compact \n" + " `dense-del` - Attributes added until the storage\n" + " is dense then an attribute deleted\n" + " the storge still in dense\n" + " `dense-del-to-compact` - Attributes added until the storage\n" + " is dense, then several attributes \n" + " deleted, the storage changed to\n" + " compact\n" + " `modify` - An attribute added then modified\n" + " `vstr` - A VL string attribute added\n" + " `remove-vstr` - A VL string attribute added then\n" + " deleted\n" + " `modify-vstr` - A VL string attribute added then \n" + " modified \n" "-q: silence printouts, few messages\n" "\n", progname); @@ -89,16 +111,6 @@ state_init(state_t *s, int argc, char **argv) unsigned long tmp; int ch; const hsize_t dims = 1; - -#if 0 - char tfile[PATH_MAX]; - char *end; - - *s = ALL_HID_INITIALIZER; - esnprintf(tfile, sizeof(tfile), "%s", argv[0]); - esnprintf(s->progname, sizeof(s->progname), "%s", HDbasename(tfile)); -#endif - char *tfile = NULL; char * end; @@ -229,7 +241,18 @@ state_init(state_t *s, int argc, char **argv) return false; } - +/* Named Pipe Subroutine: np_wr_send_receive + * Description: + * The writer sends a message to the reader, + * then waits for max_lag ticks, + * then checks the returned message from the reader. + * Return: + * True if succeed + * False if an error occurs in any step above. + * An error is mostly caused by an unexpected + * notification number from the message sent + * by the reader. + */ static bool np_wr_send_receive(state_t *s) { unsigned int i; @@ -278,6 +301,18 @@ static bool np_wr_send_receive(state_t *s) { } +/* Named Pipe Subroutine: np_rd_receive + * Description: + * The reader receives a message from the writer, + * then checks if the notification number from + * the writer is expected. + * Return: + * True if succeed + * False if an error occurs in any step above. + * An error is mostly caused by an unexpected + * notification number from the message sent + * by the writer. + */ static bool np_rd_receive(state_t *s) { /* The writer should have bumped up the value of notify. @@ -309,6 +344,13 @@ static bool np_rd_receive(state_t *s) { return false; } +/* Named Pipe Subroutine: np_rd_send + * Description: + * The reader sends an acknowledging message to the writer + * Return: + * True if succeed + * False if an error occurs in sending the message. + */ static bool np_rd_send(state_t *s) { if (HDwrite(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) { @@ -320,6 +362,15 @@ static bool np_rd_send(state_t *s) { return true; } +/* Named Pipe Subroutine: np_send_error + * Description: + * An error (notification number is 1) message is sent + * either from the reader or the writer. + * A boolean input parameter is used to choose + * either reader or writer. + * Return: + * None + */ static void np_send_error(state_t *s,bool writer) { s->np_notify = -1; if(writer) @@ -328,26 +379,55 @@ static void np_send_error(state_t *s,bool writer) { HDwrite(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)); } +/*------------------------------------------------------------------------- + * Function: add_attr + * + * Purpose: Add attributes to a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t oid + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of groups generated so far, use to generate + * newly created group name. The group name is "group-which". + * + * unsigned num_attrs + * The number of attributes to be created + * + * const char*aname_fmt + * The attribute name template used to create unique attribute names. + * + * unsigned int g_which + * This parameter is used to generate correct group name in a key + * debugging message. + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ + static bool -add_attr(state_t *s, hid_t oid,unsigned int which,unsigned num_attrs,const char*aname_fmt) { +add_attr(state_t *s, + hid_t oid, + unsigned int which, + unsigned num_attrs, + const char*aname_fmt, + unsigned int g_which) { - //char attrname[sizeof("attr-d-9999999999-999")]; char attrname[VS_ATTR_NAME_LEN]; - //char* attrname_base= "attr-%u-%u"; unsigned u; - //int i; unsigned attr_value; hid_t aid = H5I_INVALID_HID; hid_t amtype = H5I_INVALID_HID; hid_t atype = s->filetype; hid_t sid = s->one_by_one_sid; -// Just for debugging -#if 0 -if(which == 1) - goto error; -#endif - + /* Need to obtain native datatype for H5Aread */ if((amtype = H5Tget_native_type(atype,H5T_DIR_ASCEND)) <0) { H5_FAILED(); AT(); printf("H5Tget_native_type failed\n"); @@ -357,8 +437,7 @@ if(which == 1) for (u = 0; u < num_attrs; u++) { /* Create attribute */ - //HDsprintf(attrname, "attr-%u-%u", which,u); - //HDsprintf(attrname, attrname_base, which,u); + /* Construct attribute name like attr-0-0 */ HDsprintf(attrname, aname_fmt, which,u); if((aid = H5Acreate2(oid, attrname, atype, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) { @@ -373,7 +452,8 @@ if(which == 1) attr_value = u+which+1; #endif - dbgf(1, "setting attribute %s on group %u to %u\n", attrname, which, u+which); + dbgf(1, "setting attribute %s on group %u to %u\n", attrname, g_which, u+which); + /* Write data into the attribute */ if (H5Awrite(aid, amtype, &attr_value) < 0) { H5_FAILED(); AT(); @@ -388,11 +468,16 @@ if(which == 1) goto error; } + /* Writer sends a message to reader: an attribute is successfully generated. + then wait for the reader to verify and send an acknowledgement message back.*/ if (s->use_named_pipes && s->attr_test == true) { - dbgf(2, "writer: write attr - ready to send the message: %d\n", s->np_notify+1); + dbgf(2, "writer: write attr - ready to send/receive message: %d\n", s->np_notify+1); if(np_wr_send_receive(s) == false) { H5_FAILED(); AT(); dbgf(2, "writer: write attr - verification failed.\n"); + /* Note: This is (mostly) because the verification failure message + * from the reader. So don't send the error message back to + * the reader. Just stop the test. */ goto error2; } } @@ -407,6 +492,7 @@ if(which == 1) return true; error: + /* Writer needs to send an error message to the reader to stop the test*/ if(s->use_named_pipes && s->attr_test == true) np_send_error(s,true); @@ -416,7 +502,6 @@ if(which == 1) H5Tclose(amtype); } H5E_END_TRY; -///dbgf(2, "writer: LEAVE FUNC: write attr - verification failed.\n"); return false; } @@ -598,7 +683,7 @@ add_default_group_attr(state_t *s, hid_t g, unsigned int which) { const char* aname_format ="attr-%u"; - return add_attr(s,g,which,1,aname_format); + return add_attr(s,g,which,1,aname_format,which); } @@ -872,7 +957,7 @@ add_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { } /* Add attributes, until just before converting to dense storage */ - return add_attr(s,g,which,max_compact,aname_format); + return add_attr(s,g,which,max_compact,aname_format,which); error: @@ -898,7 +983,7 @@ add_attrs_compact_dense(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { /* Add another attribute, the storage becomes dense. */ if(ret_value == true) - ret_value = add_attr(s,g,which+max_compact,1,aname_format); + ret_value = add_attr(s,g,which+max_compact,1,aname_format,which); return ret_value; @@ -1290,7 +1375,7 @@ write_group(state_t *s, unsigned int which) static bool -vrfy_attr(state_t *s, hid_t g, unsigned int which, char* aname) { +vrfy_attr(state_t *s, hid_t g, unsigned int which, char* aname, unsigned int g_which) { unsigned int read_which; hid_t aid = H5I_INVALID_HID; @@ -1310,7 +1395,7 @@ vrfy_attr(state_t *s, hid_t g, unsigned int which, char* aname) { dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); } - dbgf(1, "verifying attribute %s on group %u equals %u\n", aname, which, + dbgf(1, "verifying attribute %s on group %u equals %u\n", aname, g_which, which); @@ -1463,7 +1548,7 @@ verify_default_group_attr(state_t*s,hid_t g, unsigned int which) const char* aname_format = "attr-%u"; //bool ret_value = false; HDsprintf(attrname, aname_format, which); - return vrfy_attr(s,g,which,attrname); + return vrfy_attr(s,g,which,attrname,which); } @@ -1823,7 +1908,7 @@ verify_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { for (u = 0; u < max_c; u++) { HDsprintf(attrname, aname_format, which,u); - if(false == vrfy_attr(s,g,u+which,attrname)) { + if(false == vrfy_attr(s,g,u+which,attrname,which)) { ret = false; break; } @@ -1843,7 +1928,7 @@ verify_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int whi if(ret == true) { //HDsprintf(attrname, aname_format, which,0); HDsprintf(attrname, aname_format, max_c+which,0); - ret = vrfy_attr(s,g,which+max_c,attrname); + ret = vrfy_attr(s,g,which+max_c,attrname,which); if(ret == false) dbgf(1,"verify_attrs_compact_dense failed \n"); } From 798776d6c1657371ebdf839d45eddd835f673966 Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Thu, 22 Apr 2021 14:37:44 -0500 Subject: [PATCH 04/13] Clean up code and add comments. More to do. --- test/vfd_swmr_group_writer.c | 1167 +++++++++++++++++++--------------- 1 file changed, 654 insertions(+), 513 deletions(-) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index b0957e49019..2e18169bac5 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -392,7 +392,7 @@ static void np_send_error(state_t *s,bool writer) { * HDF5 object ID (in this file: means group ID) * * unsigned int which - * The number of groups generated so far, use to generate + * The number of iterations for group creation, use to generate * newly created group name. The group name is "group-which". * * unsigned num_attrs @@ -506,187 +506,70 @@ add_attr(state_t *s, } -// Temp named pipe works. -#if 0 -static bool -add_attr(state_t *s, hid_t oid,unsigned int which,unsigned num_attrs,const char*aname_fmt,bool sl) { - - //char attrname[sizeof("attr-d-9999999999-999")]; - char attrname[VS_ATTR_NAME_LEN]; - //char* attrname_base= "attr-%u-%u"; - unsigned u; - int i; - unsigned attr_value; - hid_t aid = H5I_INVALID_HID; - hid_t amtype = H5I_INVALID_HID; - hid_t atype = s->filetype; - hid_t sid = s->one_by_one_sid; - - if((amtype = H5Tget_native_type(atype,H5T_DIR_ASCEND)) <0) { - H5_FAILED(); AT(); - printf("H5Tget_native_type failed\n"); - goto error; - } - - /* Add attributes, until just before converting to dense storage */ - for (u = 0; u < num_attrs; u++) { - - /* Create attribute */ - //HDsprintf(attrname, "attr-%u-%u", which,u); - //HDsprintf(attrname, attrname_base, which,u); - HDsprintf(attrname, aname_fmt, which,u); - if((aid = H5Acreate2(oid, attrname, atype, sid, H5P_DEFAULT, - H5P_DEFAULT)) < 0) { - H5_FAILED(); AT(); - printf("H5Acreate2 failed\n"); - goto error; - } - - attr_value = u+which; - - dbgf(1, "setting attribute %s on group %u to %u\n", attrname, which, u+which); - /* Write data into the attribute */ - if (H5Awrite(aid, amtype, &attr_value) < 0) { - H5_FAILED(); AT(); - printf("H5Awrite failed\n"); - goto error; - } - - /* Close attribute */ - if(H5Aclose(aid) < 0) { - H5_FAILED(); AT(); - printf("H5Aclose failed\n"); - goto error; - } - - if (s->attr_test == true) { - /* Bump up the value of notify to notice the reader to start to read */ - s->np_notify++; - - - dbgf(2, "writer: ready to send the message: %d.\n", s->np_notify); - - if (HDwrite(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) - err(EXIT_FAILURE, "write failed"); - - /* During the wait, writer makes repeated HDF5 API calls - * to trigger EOT at approximately the correct time */ - for(i = 0; i < s->max_lag + 1; i++) { - decisleep(s->tick_len); - H5Aexists(s->file, "nonexistent"); - } - - /* Receive the same value from the reader and verify it before - * going to the next step */ - (s->np_verify)++; - dbgf(2, "writer: ready to receive the message: %d.\n", s->np_verify); - if (HDread(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) - err(EXIT_FAILURE, "read failed"); - - dbgf(2, "writer: finish receiving the message: %d.\n", s->np_notify); - if (s->np_notify != s->np_verify) - errx(EXIT_FAILURE, "received message %d, expecting %d", s->np_notify, s->np_verify); - } - - } /* end for */ - - H5Tclose(amtype); -} -#endif - -static bool -modify_attr(state_t *s, hid_t g, const char* aname_fmt,unsigned int which) { - - //char attrname[sizeof("attr-d-9999999999-999")]; - char attrname[VS_ATTR_NAME_LEN]; - hid_t aid = H5I_INVALID_HID; - hid_t amtype = H5I_INVALID_HID; - int modify_value; - - HDsprintf(attrname,aname_fmt,which,0); - if((aid = H5Aopen(g,attrname,H5P_DEFAULT))<0) { - H5_FAILED(); AT(); - printf("H5Aopen failed\n"); - goto error; - } - - if((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND))<0) { - H5_FAILED(); AT(); - printf("H5Tget_native_type failed\n"); - goto error; - } - -#if 0 - // Add later. - //H5T_sign_t h5t_sign = H5Tget_sign(amtype); - /* Unlikely, still make sure -no overflow. */ - if((unsigned int)((int)which)!=which) { - printf("the number of %u causes of overflow when casted to an integer\n",which); - printf("number of iteration is too big, it causes overflow\n"); - goto error; - } -#endif - modify_value = which+10000; - - if (H5Awrite(aid,amtype,&modify_value) <0) { - H5_FAILED(); AT(); - printf("H5Awrite failed\n"); - goto error; - } - if (H5Tclose(amtype) < 0) { - H5_FAILED(); AT(); - goto error; - } - if (H5Aclose(aid) < 0) { - H5_FAILED(); AT(); - goto error; - } - - if (s->use_named_pipes && s->attr_test == true) { - dbgf(2, "writer: modify attr - ready to send the message: %d\n", s->np_notify+1); - if(np_wr_send_receive(s) == false) { - H5_FAILED(); AT(); - dbgf(2, "writer: write attr - verification failed.\n"); - goto error2; - } - } - - return true; -error: - if(s->use_named_pipes && s->attr_test == true) - np_send_error(s,true); - H5E_BEGIN_TRY { - H5Aclose(aid); - H5Tclose(aid); - } H5E_END_TRY; - -error2: - return false; - -} - - -#if 0 -static bool -temp_add_default_group_attr(state_t *s, hid_t g, unsigned int which) { - - const char* aname_format ="attr-%u"; - - add_attr(s,g,which,1,aname_format,false); - -} -#endif - +/*------------------------------------------------------------------------- + * Function: add_default_group_attr + * + * Purpose: Add an attribute to a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used for the "dense" storage test. + * It is also used by the group-only test. + *------------------------------------------------------------------------- +*/ static bool add_default_group_attr(state_t *s, hid_t g, unsigned int which) { const char* aname_format ="attr-%u"; + /* Note: Since we only add one attribute, the parameter for + * the number of attributes is 1. */ return add_attr(s,g,which,1,aname_format,which); } +/*------------------------------------------------------------------------- + * Function: add_vlstr_attr + * + * Purpose: Add a variable length string attribute to a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "vstr" test. + *------------------------------------------------------------------------- +*/ + static bool add_vlstr_attr(state_t*s, hid_t g, unsigned int which) { @@ -697,13 +580,15 @@ add_vlstr_attr(state_t*s, hid_t g, unsigned int which) { char *astr_val = NULL; hid_t sid = s->one_by_one_sid; + /* Allocate buffer for the VL string value */ astr_val = HDmalloc(VS_ATTR_NAME_LEN); if (astr_val == NULL) { H5_FAILED(); AT(); - printf("Allocate memory for buffer failed.\n"); + printf("Allocate memory for VL string failed.\n"); goto error; } + /* Assign the VL string value and the attribute name.. */ HDsprintf(astr_val,"%u",which); esnprintf(name, sizeof(name), "attr-%u", which); @@ -722,6 +607,7 @@ add_vlstr_attr(state_t*s, hid_t g, unsigned int which) { goto error; } + /* Generate the VL string attribute.*/ if ((aid = H5Acreate2(g, name, atype, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) { H5_FAILED(); AT(); @@ -729,16 +615,12 @@ add_vlstr_attr(state_t*s, hid_t g, unsigned int which) { goto error; } - dbgf(1, "astr_val is %s \n", astr_val); - //if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) - //if (H5Awrite(aid, H5T_NATIVE_UINT, astr_val) < 0) if (H5Awrite(aid, atype, &astr_val) < 0) { H5_FAILED(); AT(); printf("H5Awrite failed.\n"); goto error; } - if (H5Tclose(atype) < 0) { H5_FAILED(); AT(); printf("H5Tclose() failed\n"); @@ -752,6 +634,8 @@ add_vlstr_attr(state_t*s, hid_t g, unsigned int which) { HDfree(astr_val); + /* Writer sends a message to reader: a VL string attribute is successfully generated. + then wait for the reader to verify and send an acknowledgement message back. */ if (s->use_named_pipes && s->attr_test == true) { dbgf(2, "writer: write attr - ready to send the message: %d\n", s->np_notify+1); if(np_wr_send_receive(s) == false) { @@ -764,6 +648,7 @@ add_vlstr_attr(state_t*s, hid_t g, unsigned int which) { return true; error: + /* Writer needs to send an error message to the reader to stop the test*/ if(s->use_named_pipes && s->attr_test == true) np_send_error(s,true); H5E_BEGIN_TRY { @@ -778,17 +663,56 @@ add_vlstr_attr(state_t*s, hid_t g, unsigned int which) { return false; } +/*------------------------------------------------------------------------- + * Function: del_one_attr + * + * Purpose: delete one attribute in a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t obj_id + * HDF5 object ID (in this file: means group ID) + * + * bool is_dense + * if the deleted attribute is for checking the dense storage + * + * bool is_vl + * if the deleted attribute is a VL string + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute names + * according to if this attribute is a VL string or for checking + * the dense storage or the storage transition from dense to + * compact. + * + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ + static bool del_one_attr(state_t *s, hid_t obj_id,bool is_dense,bool is_vl,unsigned int which) { - //char attrname[sizeof("attr-d-9999999999-999")]; char attrname[VS_ATTR_NAME_LEN]; + + /*attribute name template for the dense storage related deletion operation */ const char* aname_format_d = "attr-d-%u-%u"; + + /*attribute name template used for general attribute deletion operation */ const char* aname_format = "attr-%u-%u"; - const char* aname_format_vl="attr-%u"; + /*attribute name template used for VL string attribute deletion operation */ + const char* aname_format_vl="attr-%u"; dbgf(2, "writer: coming to delete the attribute.\n"); + + /* Construct the attribute name */ if(is_dense == true) HDsprintf(attrname, aname_format_d, which,0); else if(is_vl == true) @@ -796,13 +720,15 @@ del_one_attr(state_t *s, hid_t obj_id,bool is_dense,bool is_vl,unsigned int whic else HDsprintf(attrname, aname_format, which,0); - /* Delete attribute */ + /* Delete the attribute */ if(H5Adelete(obj_id, attrname) <0) { H5_FAILED(); AT(); printf("H5Adelete() failed\n"); goto error; } + /* Writer sends a message to reader: an attribute is successfully generated. + then wait for the reader to verify and send an acknowledgement message back. */ if(s->use_named_pipes && s->attr_test == true) { dbgf(2, "writer: delete attr - ready to send the message: %d\n", s->np_notify+1); if(np_wr_send_receive(s) == false) { @@ -822,12 +748,40 @@ del_one_attr(state_t *s, hid_t obj_id,bool is_dense,bool is_vl,unsigned int whic return false; } +/*------------------------------------------------------------------------- + * Function: add_del_vlstr_attr + * + * Purpose: Add a variable length string attribute + * then delete this attribute in this a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "remove-vstr" test. + *------------------------------------------------------------------------- +*/ + static bool add_del_vlstr_attr(state_t *s, hid_t g, unsigned int which) { bool ret_value = false; + /* Add a VL string attribute then delete it. */ ret_value = add_vlstr_attr(s,g,which); if(ret_value == true) ret_value = del_one_attr(s,g,false,true,which); @@ -836,25 +790,142 @@ add_del_vlstr_attr(state_t *s, hid_t g, unsigned int which) { } +/*------------------------------------------------------------------------- + * Function: modify_attr + * + * Purpose: Modify the value of an attribute in a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * const char*aname_fmt + * The attribute name template used to create unique attribute names. + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group name. The group name is "group-which". + * + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ + + +static bool +modify_attr(state_t *s, hid_t g, const char* aname_fmt,unsigned int which) { + + char attrname[VS_ATTR_NAME_LEN]; + hid_t aid = H5I_INVALID_HID; + hid_t amtype = H5I_INVALID_HID; + unsigned int modify_value; + + HDsprintf(attrname,aname_fmt,which,0); + if((aid = H5Aopen(g,attrname,H5P_DEFAULT))<0) { + H5_FAILED(); AT(); + printf("H5Aopen failed\n"); + goto error; + } + + if((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND))<0) { + H5_FAILED(); AT(); + printf("H5Tget_native_type failed\n"); + goto error; + } + + /* Make a large number to verify the change easily */ + modify_value = which+10000; + + if (H5Awrite(aid,amtype,&modify_value) <0) { + H5_FAILED(); AT(); + printf("H5Awrite failed\n"); + goto error; + } + if (H5Tclose(amtype) < 0) { + H5_FAILED(); AT(); + goto error; + } + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + goto error; + } + + /* Writer sends a message to reader: an attribute is successfully modified. + then wait for the reader to verify and send an acknowledgement message back.*/ + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: modify attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write attr - verification failed.\n"); + /* Note: This is (mostly) because the verification failure message + * from the reader. So don't send the error message back to + * the reader. Just stop the test. */ + goto error2; + } + } + + return true; +error: + /* Writer needs to send an error message to the reader to stop the test*/ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(aid); + } H5E_END_TRY; + +error2: + return false; + +} + +/*------------------------------------------------------------------------- + * Function: modify_vlstr_attr + * + * Purpose: Modify the value of an VL string attribute in a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group name. The group name is "group-which". + * + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ + + + static bool modify_vlstr_attr(state_t*s,hid_t g, unsigned int which) { hid_t aid = H5I_INVALID_HID; hid_t atype = H5I_INVALID_HID; - //char name[sizeof("attr-9999999999")]; char name[VS_ATTR_NAME_LEN]; char *astr_val = NULL; - //astr_val = malloc(sizeof("9999999999!")); astr_val = HDmalloc(VS_ATTR_NAME_LEN); if (astr_val == NULL) { H5_FAILED(); AT(); - printf("Allocate memory for buffer failed.\n"); + printf("Allocate memory for VL string failed.\n"); goto error; } + /* Change the VL string value and create the attribute name. */ HDsprintf(astr_val,"%u%c",which,'A'); - //const char *astr_val="test"; esnprintf(name, sizeof(name), "attr-%u", which); dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); @@ -872,17 +943,15 @@ modify_vlstr_attr(state_t*s,hid_t g, unsigned int which) { goto error; } - //if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, + /* Open this attribute. */ if ((aid = H5Aopen(g, name, H5P_DEFAULT))<0) { H5_FAILED(); AT(); printf("H5Aopen failed.\n"); goto error; } - dbgf(1, "astr_val is %s \n", astr_val); + dbgf(1, "The modified VL string value is %s \n", astr_val); - //if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) - //if (H5Awrite(aid, H5T_NATIVE_UINT, astr_val) < 0) if (H5Awrite(aid, atype, &astr_val) < 0) { H5_FAILED(); AT(); printf("H5Awrite failed.\n"); @@ -903,6 +972,8 @@ modify_vlstr_attr(state_t*s,hid_t g, unsigned int which) { HDfree(astr_val); + /* Writer sends a message to reader: a VL string attribute is successfully generated. + then wait for the reader to verify and send an acknowledgement message back. */ if (s->use_named_pipes && s->attr_test == true) { dbgf(2, "writer: modify vl attr - ready to send the message: %d\n", s->np_notify+1); if(np_wr_send_receive(s) == false) { @@ -931,11 +1002,37 @@ modify_vlstr_attr(state_t*s,hid_t g, unsigned int which) { } +/*------------------------------------------------------------------------- + * Function: add_modify_vlstr_attr + * + * Purpose: Add a variable length string attribute + * then modify this attribute in this a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "modify-vstr" test. + *------------------------------------------------------------------------- +*/ + static bool add_modify_vlstr_attr(state_t *s, hid_t g, unsigned int which) { bool ret_value = false; - //const char* aname_format ="attr-%u"; ret_value = add_vlstr_attr(s,g,which); if (true == ret_value) ret_value = modify_vlstr_attr(s,g,which); @@ -943,6 +1040,39 @@ add_modify_vlstr_attr(state_t *s, hid_t g, unsigned int which) { return ret_value; } +/*------------------------------------------------------------------------- + * Function: add_attrs_compact + * + * Purpose: Add some attributes to the group. + * the number of attributes should be the maximal number of + * attributes that the compact storage can hold + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "modify-vstr" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ + static bool add_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { @@ -950,20 +1080,56 @@ add_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { unsigned min_dense = 0; const char* aname_format="attr-%u-%u"; + /* Obtain the maximal number of attributes to be stored in compact + * storage and the minimal number of attributes to be stored in + * dense storage. */ if(H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense)<0) { H5_FAILED(); AT(); printf("H5Pget_attr_phase_change() failed\n"); goto error; } - /* Add attributes, until just before converting to dense storage */ + /* Add max_compact attributes, these attributes are stored in + * compact storage. */ return add_attr(s,g,which,max_compact,aname_format,which); - error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); return false; } +/*------------------------------------------------------------------------- + * Function: add_attrs_compact_dense + * + * Purpose: Add some attributes to the group. + * First, the number of attributes should be the maximal number + * of attributes that the compact storage can hold. + * Then, add another atribute, the storage becomes dense. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "compact-to-dense" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ + static bool add_attrs_compact_dense(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { @@ -988,53 +1154,73 @@ add_attrs_compact_dense(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { return ret_value; error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); return false; } -#if 0 -static bool -del_one_attr(state_t *s, hid_t obj_id,bool is_dense,unsigned int which) { - - char attrname[sizeof("attr-d-9999999999-999")]; - const char* aname_format_d = "attr-d-%u-%u"; - const char* aname_format = "attr-%u-%u"; - - if(is_dense == true) - HDsprintf(attrname, aname_format_d, which,0); -//printf("attrname is %s\n",attrname); - else - HDsprintf(attrname, aname_format, which,0); - - /* Delete attribute */ - H5Adelete(obj_id, attrname); - nanosleep(&(s->update_interval), NULL); - - -} -#endif +/*------------------------------------------------------------------------- + * Function: del_attrs_compact_dense_compact + * + * Purpose: delete some attributes in the group. + * The number of attributes are deleted in such a way + * that the attribute storage changes from compact to + * dense then to compact again. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * Return: Success: true + * Failure: false + * + * Note: This is an internal function used by the + * "dense-del-to-compact" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ static bool -del_attrs_compact_dense_compact(state_t *s, hid_t obj_id,hid_t gcpl,unsigned int which) { +del_attrs_compact_dense_compact(state_t *s, + hid_t obj_id, + hid_t gcpl, + unsigned int which) { unsigned max_compact = 0; unsigned min_dense = 0; unsigned u = 0; - //char attrname[sizeof("attr-d-9999999999-999")]; char attrname[VS_ATTR_NAME_LEN]; const char* aname_format="attr-%u-%u"; const char* adname_format="attr-d-%u-%u"; + /* Obtain the maximal number of attributes to be stored in compact + * storage and the minimal number of attributes to be stored in + * dense storage. */ if (H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense) < 0) { H5_FAILED(); AT(); printf("H5Pget_attr_phase_change failed\n"); goto error; } u= max_compact +1; + #if 0 if(max_compact < min_dense) printf("Minimum number of attributes stored in dense storage should be less than maximum number of attributes stored in compact storage.\n"); #endif + + // delete a number of attributes so that the attribute storage just becomes dense. for(u--;u>=(min_dense-1);u--) { HDsprintf(attrname, aname_format, which,max_compact-u); if (H5Adelete(obj_id,attrname) < 0) { @@ -1042,6 +1228,11 @@ del_attrs_compact_dense_compact(state_t *s, hid_t obj_id,hid_t gcpl,unsigned int printf("H5Adelete failed\n"); goto error; } + + /* For each attribute deletion, we want to ensure the verification + * from the reader. + * So writer sends a message to reader: an attribute is successfully deleted. + then wait for reader to verify and send an acknowledgement message back. */ if(s->use_named_pipes && s->attr_test == true) { dbgf(2, "writer: delete attr - ready to send the message: %d\n", s->np_notify+1); if(np_wr_send_receive(s) == false) { @@ -1052,14 +1243,21 @@ del_attrs_compact_dense_compact(state_t *s, hid_t obj_id,hid_t gcpl,unsigned int } } - // The writer only deletes the attribute attr-which-0 + /* The writer deletes another attribute, the storage is + * still dense. However, the attribute to be deleted + * doesn't follow the previous for loop. It may be + * in different location in the object header. Just add + * a litter variation to check if this operation is successful. + * The attribute name to be deleted is attr-max_compact+which-0 + */ + HDsprintf(attrname,adname_format,max_compact+which,0); - /// CHECK HERE, add H5Adelete() if (H5Adelete(obj_id,attrname) < 0) { H5_FAILED(); AT(); printf("H5Adelete failed\n"); goto error; } + /* Again we need to notify the reader via Named pipe. */ if(s->use_named_pipes && s->attr_test == true) { dbgf(2, "writer: delete attr - ready to send the message: %d\n", s->np_notify+1); if(np_wr_send_receive(s) == false) { @@ -1069,13 +1267,16 @@ del_attrs_compact_dense_compact(state_t *s, hid_t obj_id,hid_t gcpl,unsigned int } } - - -// May H5Oget_info3 -- obtain the number of attributes. -//Check the number of attributes >=min_dense. -//We may use the internal function -//is_dense = H5O__is_attr_dense_test(dataset) to check if it is dense in the future. -// + /* The following comments are left here in case in the future we want to + * use HDF5 function to verify the attribute storage */ +#if 0 + // May H5Oget_info3 -- obtain the number of attributes. + //Check the number of attributes >=min_dense. + //We may use the internal function + //is_dense = H5O__is_attr_dense_test(dataset) to check if it is dense in the future. + // +#endif + return true; error: @@ -1086,6 +1287,36 @@ del_attrs_compact_dense_compact(state_t *s, hid_t obj_id,hid_t gcpl,unsigned int return false; } +/*------------------------------------------------------------------------- + * Function: add_del_attrs_compact + * + * Purpose: Add some attributes to the group and then delete one attribute. + * First, the number of attributes to be added should be the + * maximal number of attributes that the compact storage can hold. + * Then, delete one atribute, the storage is still compact. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "compact-del" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ static bool add_del_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { @@ -1101,6 +1332,39 @@ add_del_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { } +/*------------------------------------------------------------------------- + * Function: add_del_attrs_compact_dense + * + * Purpose: Add some attributes to the group and then delete one attribute. + * First, the number of attributes to be added exceeds + * the maximal number of attributes that the compact storage can hold. + * The storage changes from compact to dense. + * Then, delete one atribute, the storage is still dense. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "dense-del" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ + + static bool add_del_attrs_compact_dense(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { @@ -1121,21 +1385,87 @@ add_del_attrs_compact_dense(state_t *s, hid_t g, hid_t gcpl, unsigned int which) return ret_value; error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); return false; } + +/*------------------------------------------------------------------------- + * Function: add_del_attrs_compact_dense_compact + * + * Purpose: Add attributes to the group and then delete some of them. + * First, the number of attributes to be added exceeds + * the maximal number of attributes that the compact storage can hold. + * The storage changes from compact to dense. + * Then, delete some attributes, the storage changes from + * dense to compact again. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "dense-del-to-compact" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ + + static bool -add_del_attrs_compact_dense_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { +add_del_attrs_compact_dense_compact(state_t *s, + hid_t g, + hid_t gcpl, + unsigned int which) { bool ret_value = false; ret_value = add_attrs_compact_dense(s,g,gcpl,which); if(ret_value == true) ret_value = del_attrs_compact_dense_compact(s,g,gcpl,which); - - return ret_value; -} + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: add_modify_default_group_attr + * + * Purpose: Add an attribute then modify the value to a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used for the "modify" storage test. + *------------------------------------------------------------------------- +*/ + static bool add_modify_default_group_attr(state_t *s, hid_t g, unsigned int which) { @@ -1149,6 +1479,34 @@ add_modify_default_group_attr(state_t *s, hid_t g, unsigned int which) { } +/*------------------------------------------------------------------------- + * Function: add_group_attribute + * + * Purpose: Check the attribute test pattern and then call the + * correponding test function.. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This is called by the write_group() function. + *------------------------------------------------------------------------- +*/ + static bool add_group_attribute(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { @@ -1190,30 +1548,30 @@ add_group_attribute(state_t *s, hid_t g, hid_t gcpl, unsigned int which) ret_value = add_default_group_attr(s, g, which); break; } -//printf("add_group_attribute return value %d\n",(int)ret_value); return ret_value; } -#if 0 -{ - hid_t aid; - char name[sizeof("attr-9999999999")]; - - esnprintf(name, sizeof(name), "attr-%u", which); - - dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); - - if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, - H5P_DEFAULT)) < 0) - errx(EXIT_FAILURE, "H5Acreate2 failed"); - - if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) - errx(EXIT_FAILURE, "H5Awrite failed"); - if (H5Aclose(aid) < 0) - errx(EXIT_FAILURE, "H5Aclose failed"); -} -#endif +/*------------------------------------------------------------------------- + * Function: write_group + * + * Purpose: Create a group and carry out attribute operations(add,delete etc.) + * according to the attribute test pattern. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * unsigned int which + * The number of iterations for group creation + * + * + * Return: Success: true + * Failure: false + * + * Note: This is called by the main() function. + *------------------------------------------------------------------------- +*/ static bool write_group(state_t *s, unsigned int which) @@ -1223,7 +1581,6 @@ write_group(state_t *s, unsigned int which) hid_t gcpl = H5I_INVALID_HID; bool result = true; - //assert(which < s->nsteps); if (which >= s->nsteps) { H5_FAILED(); AT(); printf("Number of created groups is out of bounds\n"); @@ -1252,29 +1609,30 @@ write_group(state_t *s, unsigned int which) H5P_DEFAULT)) < 0) { H5_FAILED(); AT(); printf("H5Gcreate2 failed\n"); - //np_send_error(s,true); goto error; } - /* If an attribute test is turned on, the NP writer sends a message */ + /* If an attribute test is turned on and named pipes are used, + * the writer should send and receive messages after the group creation. + * This will distinguish an attribute operation error from an + * group creation error. + * Writer sends a message to reader: an attribute is successfully generated. + * then wait for the reader to verify and send an acknowledgement message back.*/ if (s->use_named_pipes && s->attr_test == true) { dbgf(2, "writer: ready to send the message: %d\n", s->np_notify+1); if(np_wr_send_receive(s) == false) { H5_FAILED(); AT(); + /* Note: This is (mostly) because the verification failure message + * from the reader. So don't send the error message back to + * the reader. Just stop the test. */ goto error2; } } -#if 0 -dbgf(1,"Writer: pass group creation\n"); -#endif + + /* Then carry out the attribute operation. */ if (s->asteps != 0 && which % s->asteps == 0) result = add_group_attribute(s, g, gcpl,which); -#if 0 -if(result == true) -printf("Group: successfully receiving the verification from the reader.\n"); -else -printf("Group: Fail to receive the verficiation from the reader.\n"); -#endif + if (H5Gclose(g) < 0) { H5_FAILED(); AT(); printf("H5Gclose failed\n"); @@ -1290,8 +1648,7 @@ printf("Group: Fail to receive the verficiation from the reader.\n"); return result; error: - // Consistent - // But if receiving an error message,no need to send the error again. + /* Writer needs to send an error message to the reader to stop the test*/ if(s->use_named_pipes && s->attr_test == true) np_send_error(s,true); @@ -1306,86 +1663,46 @@ printf("Group: Fail to receive the verficiation from the reader.\n"); } -// Temp name pipe works. -#if 0 -static bool -write_group(state_t *s, unsigned int which) -{ - char name[sizeof("/group-9999999999")]; - hid_t g; - hid_t gcpl; - int i; - - assert(which < s->nsteps); - - esnprintf(name, sizeof(name), "/group-%d", which); - gcpl = H5Pcreate(H5P_GROUP_CREATE); - if(gcpl <0) - errx(EXIT_FAILURE, "H5Pcreate failed"); - if(s->at_pattern =='d') - H5Pset_attr_phase_change(gcpl, 0, 0); - - g = H5Gcreate2(s->file, name, H5P_DEFAULT, gcpl, H5P_DEFAULT); - // TODO: before the failure, needs to check if the reader is waiting for the pipe. - if (g < 0) - errx(EXIT_FAILURE, "H5Gcreate(, \"%s\", ) failed", name); - if (s->attr_test == true) { - /* Bump up the value of notify to notice the reader to start to read */ - s->np_notify++; - - - dbgf(2, "writer: ready to send the message: %d.\n", s->np_notify); - - if (HDwrite(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) - err(EXIT_FAILURE, "write failed"); - - /* During the wait, writer makes repeated HDF5 API calls - * to trigger EOT at approximately the correct time */ - for(i = 0; i < s->max_lag + 1; i++) { - decisleep(s->tick_len); - H5Aexists(s->file, "nonexistent"); - } - - /* Receive the same value from the reader and verify it before - * going to the next step */ - (s->np_verify)++; - dbgf(2, "writer: ready to receive the message: %d.\n", s->np_verify); - if (HDread(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) - err(EXIT_FAILURE, "read failed"); - - dbgf(2, "writer: finish receiving the message: %d.\n", s->np_notify); - if (s->np_notify != s->np_verify) - errx(EXIT_FAILURE, "received message %d, expecting %d", s->np_notify, s->np_verify); - } - - if (s->asteps != 0 && which % s->asteps == 0) { - add_group_attribute(s, g, gcpl,which); - } - - - if(H5Pclose(gcpl) <0) - errx(EXIT_FAILURE, "H5Pcreate failed"); - - if (H5Gclose(g) < 0) - errx(EXIT_FAILURE, "H5Gclose failed"); - -} - -#endif - +/*------------------------------------------------------------------------- + * Function: vrfy_attr + * + * Purpose: Verify is a group attribute value is as expected.. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t oid + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group name. The group name is "group-which". + * + * const char*aname_fmt + * The attribute name template used to create unique attribute names. + * + * unsigned int g_which + * This parameter is used to generate correct group name in a key + * debugging message. + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ static bool -vrfy_attr(state_t *s, hid_t g, unsigned int which, char* aname, unsigned int g_which) { +vrfy_attr(state_t *s, + hid_t g, + unsigned int which, + const char* aname, + unsigned int g_which) { unsigned int read_which; hid_t aid = H5I_INVALID_HID; hid_t amtype = H5I_INVALID_HID; - //char name[sizeof("attr-d-9999999999-999")]; - - //esnprintf(name, sizeof(name), "attr-%u", which); - //esnprintf(name, sizeof(name), aname_fmt, which); - if(s->use_named_pipes && true == s->attr_test) { if(false == np_rd_receive(s)) { H5_FAILED(); AT(); @@ -1398,16 +1715,6 @@ vrfy_attr(state_t *s, hid_t g, unsigned int which, char* aname, unsigned int g_ dbgf(1, "verifying attribute %s on group %u equals %u\n", aname, g_which, which); - -#if 0 - if (H5Sget_simple_extent_npoints(s->one_by_one_sid)!=1) { - dbgf(1, "The number of elements of %s on group %u should be 1, exit.\n" - name,which); - restore_estack(es); - return false; - } -#endif - if ((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND)) <0) { H5_FAILED(); AT(); printf("H5Tget_native_type failed\n"); @@ -1459,94 +1766,11 @@ vrfy_attr(state_t *s, hid_t g, unsigned int which, char* aname, unsigned int g_ } -// Temp name pipe works -#if 0 -static bool -vrfy_attr(state_t *s, hid_t g, unsigned int which, char* aname, bool sl) { - - estack_state_t es; - unsigned int read_which; - hid_t aid; - hid_t amtype; - bool ret_value = true; - //char name[sizeof("attr-d-9999999999-999")]; - - //esnprintf(name, sizeof(name), "attr-%u", which); - //esnprintf(name, sizeof(name), aname_fmt, which); - dbgf(1, "verifying attribute %s on group %u equals %u\n", aname, which, - which); - - if(true == s->attr_test) { - s->np_verify++; - /* Receive the notify that the writer bumped up the value */ - if (HDread(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) - err(EXIT_FAILURE, "read failed"); - if (s->np_notify != s->np_verify) - errx(EXIT_FAILURE, "received message %d, expecting %d", s->np_notify, s->np_verify); - decisleep(3*(s->tick_len)); - dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); - - } - - es = disable_estack(); - -#if 0 - if (H5Sget_simple_extent_npoints(s->one_by_one_sid)!=1) { - dbgf(1, "The number of elements of %s on group %u should be 1, exit.\n" - name,which); - restore_estack(es); - return false; - } -#endif - - amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND); - if ((aid = H5Aopen(g, aname, H5P_DEFAULT)) < 0) { - restore_estack(es); - ret_value = false; - } - - if(ret_value == true) { - if (H5Aread(aid, amtype, &read_which) < 0) { - restore_estack(es); - if (H5Aclose(aid) < 0) - errx(EXIT_FAILURE, "H5Aclose failed"); - ret_value = false; - } - } - - restore_estack(es); - - if (H5Aclose(aid) < 0) - errx(EXIT_FAILURE, "H5Aclose failed"); - - if(ret_value == true) - ret_value = (read_which == which); - - if(s->attr_test == true) { - if(ret_value == false) { - dbgf(2, "reader: the add_attribute verfication failed %d\n", which); - dbgf(2, "reader: the add_attribute verfication failed, the value is %d\n", read_which); - - s->np_notify = 0; - } - if (HDwrite(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) - err(EXIT_FAILURE, "write failed"); - - dbgf(2, "reader: finish sending back the message: %d\n.", s->np_notify); - } - - return ret_value; - -} -#endif - static bool verify_default_group_attr(state_t*s,hid_t g, unsigned int which) { - //char attrname[sizeof("attr-9999999999")]; char attrname[VS_ATTR_NAME_LEN]; const char* aname_format = "attr-%u"; - //bool ret_value = false; HDsprintf(attrname, aname_format, which); return vrfy_attr(s,g,which,attrname,which); @@ -1558,11 +1782,9 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { bool ret = false; const char* aname_fmt ="attr-%u"; - estack_state_t es; - int read_which; + unsigned int read_which; hid_t aid = H5I_INVALID_HID; hid_t amtype = H5I_INVALID_HID; - //char aname[sizeof("attr-d-9999999999-999")]; char attrname[VS_ATTR_NAME_LEN]; ret = verify_default_group_attr(s,g,which); @@ -1578,17 +1800,6 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { } esnprintf(attrname, sizeof(attrname), aname_fmt, which); - //dbgf(1, "verifying attribute %s on group %u equals %u\n", aname, which, - // which); - -#if 0 - if (H5Sget_simple_extent_npoints(s->one_by_one_sid)!=1) { - dbgf(1, "The number of elements of %s on group %u should be 1, exit.\n" - name,which); - restore_estack(es); - return false; - } -#endif if ((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND)) < 0) { H5_FAILED(); AT(); @@ -1620,15 +1831,6 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { goto error; } -#if 0 - if((unsigned int)((int)which)!=which) { - H5_FAILED(); AT(); - printf("the unsigned %u causes overflow when casted to signed.\n",which); - printf("number of iteration is too big, it causes overflow.\n"); - goto error; - } -#endif - if(read_which != (which+10000)) { H5_FAILED(); AT(); dbgf(2, "reader: the modified_attr() expected value is %d\n", (-1)*(int)which); @@ -1661,30 +1863,15 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { return false; } -#if 0 -static bool -verify_default_group_attr(state_t*s,hid_t g, unsigned int which) -{ - char attrname[sizeof("attr-9999999999")]; - const char* aname_format = "attr-%u"; - HDsprintf(attrname, aname_format, which); - return vrfy_attr(s,g,which,attrname,false); -} -#endif - static bool verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) { - estack_state_t es; - //unsigned int read_which; - bool ret = false; hid_t aid = H5I_INVALID_HID; hid_t atype = H5I_INVALID_HID; - //char name[sizeof("attr-9999999999")]; char name[VS_ATTR_NAME_LEN]; - char *astr_val_exp = NULL; - char * astr_val = NULL; + char *astr_val_exp; + char * astr_val; if(s->use_named_pipes && true == s->attr_test) { if(false == np_rd_receive(s)) { @@ -1695,15 +1882,6 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); } -#if 0 - astr_val = malloc(VS_ATTR_NAME_LEN); - if (astr_val == NULL) { - H5_FAILED(); AT(); - printf("Allocate memory for buffer failed.\n"); - goto error; - } -#endif - astr_val_exp = HDmalloc(VS_ATTR_NAME_LEN); if (astr_val_exp == NULL) { H5_FAILED(); AT(); @@ -1722,7 +1900,6 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) dbgf(1,"expected vl attr is= %s\n",astr_val_exp); - es = disable_estack(); if ((aid = H5Aopen(g, name, H5P_DEFAULT)) < 0) { H5_FAILED(); AT(); printf("H5Aopen failed\n"); @@ -1757,8 +1934,6 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) goto error; } - //restore_estack(es); - if(H5Tclose(atype) <0) { H5_FAILED(); AT(); printf("H5Tclose failed.\n"); @@ -1803,7 +1978,6 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) static bool verify_del_one_attr(state_t *s,hid_t g, const char *aname) { - //bool ret = false; htri_t attr_exists = FALSE; if(s->use_named_pipes && true == s->attr_test) { @@ -1815,7 +1989,6 @@ verify_del_one_attr(state_t *s,hid_t g, const char *aname) { dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); } - attr_exists = H5Aexists_by_name(g,".",aname,H5P_DEFAULT); if(attr_exists == FALSE) { dbgf(1,"verify_del_attrs_compact() test: \n"); @@ -1848,38 +2021,14 @@ verify_del_one_attr(state_t *s,hid_t g, const char *aname) { static bool verify_remove_vlstr_attr(state_t* s,hid_t g, unsigned int which) { - estack_state_t es; bool ret = false; - htri_t attr_exists = FALSE; - //char attrname[sizeof("attr-9999999999")]; char attrname[VS_ATTR_NAME_LEN]; const char* aname_format = "attr-%u"; ret = verify_group_vlstr_attr(s,g,which,false); if(ret == true) { HDsprintf(attrname,aname_format,which); - // Add error handling later. ret = verify_del_one_attr(s,g,attrname); -#if 0 - es = disable_estack(); - attr_exists = H5Aexists_by_name(g,".",attrname,H5P_DEFAULT); - restore_estack(es); - - if(attr_exists == FALSE) { - dbgf(1,"verify_remove_vlstr_attr test: \n"); - dbgf(1," attribute %s on group %u is successfully deleted. \n",attrname,which); - ret = true; - } - else if(attr_exists == TRUE) { - dbgf(1,"verify_remove_vlstr_attr test failed \n"); - ret = false; - } - else{ - dbgf(1,"H5Aexists_by_name failed \n"); - ret = false; - } - -#endif } return ret; } @@ -1889,7 +2038,6 @@ verify_modify_vlstr_attr(state_t *s, hid_t g, unsigned int which){ bool ret = false; - // May change the sid with state_t s ret = verify_group_vlstr_attr(s,g,which,false); if(ret == true) ret = verify_group_vlstr_attr(s,g,which,true); @@ -1903,7 +2051,6 @@ verify_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { unsigned u; bool ret = true; const char* aname_format = "attr-%u-%u"; - //char attrname[sizeof("attr-9999999999-999")]; char attrname[VS_ATTR_NAME_LEN]; for (u = 0; u < max_c; u++) { @@ -1938,8 +2085,6 @@ verify_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int whi static bool verify_del_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { - estack_state_t es; - htri_t attr_exists = FALSE; const char* aname_format = "attr-%u-%u"; //char attrname[sizeof("attr-d-9999999999-999")]; char attrname[VS_ATTR_NAME_LEN]; @@ -1975,8 +2120,6 @@ verify_del_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which static bool verify_del_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int which) { - estack_state_t es; - htri_t attr_exists = FALSE; const char* aname_format = "attr-d-%u-%u"; //char attrname[sizeof("attr-d-9999999999-999")]; char attrname[VS_ATTR_NAME_LEN]; @@ -2014,9 +2157,7 @@ verify_del_attrs_compact_dense_compact(state_t *s, unsigned max_c, unsigned min_d, unsigned int which) { - estack_state_t es; unsigned u; - htri_t attr_exists = FALSE; const char* aname_format = "attr-%u-%u"; char attrname[VS_ATTR_NAME_LEN]; //char attrname[sizeof("attr-9999999999-999")]; From 2afc32cc138f83401623a4108ae99047f32e23e7 Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Thu, 22 Apr 2021 16:16:26 -0500 Subject: [PATCH 05/13] More comments and shorten the test time a bit --- test/testvfdswmr.sh.in | 4 +- test/vfd_swmr_group_writer.c | 1295 +++++++--------------------------- 2 files changed, 265 insertions(+), 1034 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index e287e79c26b..1a1d6837bb7 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -62,7 +62,7 @@ if [[ "$HDF5TestExpress" -eq 0 ]] ; then # Setting for exhaustive run BIGSET_few_s=40 BIGSET_many_s=1000 GROUP_n=400 - GROUP_attr_n=10 + GROUP_attr_n=4 elif [[ "$HDF5TestExpress" -gt 1 ]]; then # Setting for quick run BIGSET_n=10 BIGSET_few_s=10 @@ -669,7 +669,6 @@ grp_attr_list=( grp_sub_attr_list=( "dense-del-to-compact" "modify" - "vstr" "remove-vstr" "modify-vstr" ) @@ -678,7 +677,6 @@ grp_short_sub_attr_list=( "dense" "compact-del" "modify" - "vstr" "remove-vstr" "modify-vstr" ) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 2e18169bac5..4627dc69a7a 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -346,7 +346,7 @@ static bool np_rd_receive(state_t *s) { /* Named Pipe Subroutine: np_rd_send * Description: - * The reader sends an acknowledging message to the writer + * The reader sends an acknowledgement to the writer * Return: * True if succeed * False if an error occurs in sending the message. @@ -1666,7 +1666,7 @@ write_group(state_t *s, unsigned int which) /*------------------------------------------------------------------------- * Function: vrfy_attr * - * Purpose: Verify is a group attribute value is as expected.. + * Purpose: Verify is a group attribute value is as expected. * * Parameters: state_t *s * The struct that stores information of HDF5 file, named pipe @@ -1679,8 +1679,8 @@ write_group(state_t *s, unsigned int which) * The number of iterations for group creation, use to generate * newly created group name. The group name is "group-which". * - * const char*aname_fmt - * The attribute name template used to create unique attribute names. + * const char*aname + * The attribute name * * unsigned int g_which * This parameter is used to generate correct group name in a key @@ -1703,15 +1703,22 @@ vrfy_attr(state_t *s, hid_t aid = H5I_INVALID_HID; hid_t amtype = H5I_INVALID_HID; + /* The reader receives a message from the writer.Then sleep + * for a few ticks or stop the test if receiving an error + * message. + */ if(s->use_named_pipes && true == s->attr_test) { if(false == np_rd_receive(s)) { H5_FAILED(); AT(); + /* Since receiving the error message from the writer, + * just stop the test. */ goto error2; } decisleep(s->tick_len * s->update_interval); dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); } + /* Go ahead to read the attribute. */ dbgf(1, "verifying attribute %s on group %u equals %u\n", aname, g_which, which); @@ -1747,6 +1754,7 @@ vrfy_attr(state_t *s, goto error; } + /* If the read value is expected, send back an OK message to the writer. */ if(s->use_named_pipes && s->attr_test == true) { if(np_rd_send(s)==false) goto error; @@ -1759,6 +1767,8 @@ vrfy_attr(state_t *s, H5Tclose(amtype); H5Aclose(aid); } H5E_END_TRY; + + /* Send back an error message to the writer so that the writer can stop. */ if(s->use_named_pipes && s->attr_test == true) np_send_error(s,false); error2: @@ -1766,6 +1776,31 @@ vrfy_attr(state_t *s, } +/*------------------------------------------------------------------------- + * Function: verify_default_group_attr + * + * Purpose: Check if the reader can retrieve the correct value of a + * group attribute corrected by the writer. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The expected attribute value. It is also used to construct the + * group name. + * + * Return: Success: true + * Failure: false + * + * Note: This function is used for the "dense" storage test. + * It is also used by the group-only test. + *------------------------------------------------------------------------- +*/ + static bool verify_default_group_attr(state_t*s,hid_t g, unsigned int which) { @@ -1776,6 +1811,31 @@ verify_default_group_attr(state_t*s,hid_t g, unsigned int which) } +/*------------------------------------------------------------------------- + * Function: verify_modify_attr + * + * Purpose: Check if the reader can retrieve the correct value of + * an attribute in a group, first the original value then + * the modified value. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The expected attribute value. It is also used to construct the + * group name. The modified attribute value can be derived from + * the expected attribute value. + * + * Return: Success: true + * Failure: false + * + * Note: This function is used for the "modified" test. + *------------------------------------------------------------------------- +*/ static bool verify_modify_attr(state_t *s, hid_t g, unsigned int which) { @@ -1787,9 +1847,16 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { hid_t amtype = H5I_INVALID_HID; char attrname[VS_ATTR_NAME_LEN]; + /* First verify the original attribute value */ ret = verify_default_group_attr(s,g,which); + /* Then the modified value */ if(ret == true) { + + /* The reader receives a message from the writer.Then sleep + * for a few ticks or stop the test if receiving an error + * message. + */ if(s->use_named_pipes && true == s->attr_test) { if(false == np_rd_receive(s)) { H5_FAILED(); AT(); @@ -1799,8 +1866,8 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); } + /* Go ahead to read the attribute. */ esnprintf(attrname, sizeof(attrname), aname_fmt, which); - if ((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND)) < 0) { H5_FAILED(); AT(); printf("H5Tget_native_type failed\n"); @@ -1831,6 +1898,7 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { goto error; } + /* verify the modified value */ if(read_which != (which+10000)) { H5_FAILED(); AT(); dbgf(2, "reader: the modified_attr() expected value is %d\n", (-1)*(int)which); @@ -1839,6 +1907,7 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { goto error; } + /* The reader sends an OK message back to the writer. */ if(s->use_named_pipes && s->attr_test == true) { if(np_rd_send(s)==false) goto error2; @@ -1855,6 +1924,7 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { H5Tclose(amtype); } H5E_END_TRY; + /* The reader needs to send an error message back to the writer to stop the test.*/ if(s->use_named_pipes && s->attr_test == true) np_send_error(s,false); @@ -1863,6 +1933,36 @@ verify_modify_attr(state_t *s, hid_t g, unsigned int which) { return false; } +/*------------------------------------------------------------------------- + * Function: verify_group_vlstr_attr + * + * Purpose: Check if the reader can retrieve the correct value of + * a variable length string attribute created by the writer. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * Use to derieve the expected attribute value. It is also used + * to construct the group name. + * + * bool vrfy_mod + * true if this function is used for the modified VL string test. + * false if this function is just used for the VL string test. + * + * Return: Success: true + * Failure: false + * + * Note: This function is an internal function used by + * both the "vlstr" and the "modify-vstr" tests. + *------------------------------------------------------------------------- +*/ + + static bool verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) { @@ -1873,6 +1973,10 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) char *astr_val_exp; char * astr_val; + /* The reader receives a message from the writer.Then sleep + * for a few ticks or stop the test if the received message + * is an error message. + */ if(s->use_named_pipes && true == s->attr_test) { if(false == np_rd_receive(s)) { H5_FAILED(); AT(); @@ -1882,6 +1986,7 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); } + /* Go ahead to read the VL string attribute. */ astr_val_exp = HDmalloc(VS_ATTR_NAME_LEN); if (astr_val_exp == NULL) { H5_FAILED(); AT(); @@ -1890,6 +1995,9 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) } esnprintf(name, sizeof(name), "attr-%u", which); + + /* Construct the expected VL string value,depending if + * it is the modified value or the original value. */ if(vrfy_mod == true) HDsprintf(astr_val_exp,"%u%c",which,'A'); else @@ -1906,7 +2014,7 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) goto error; } - /* Create a datatype to refer to. */ + /* Create a VL string datatype */ if ((atype = H5Tcopy(H5T_C_S1)) < 0) { H5_FAILED(); AT(); printf("Cannot create variable length datatype.\n"); @@ -1949,6 +2057,7 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) H5free_memory(astr_val); HDfree(astr_val_exp); + /* Reader sends an OK message back to the reader */ if(s->use_named_pipes && s->attr_test == true) { if(np_rd_send(s)==false) goto error2; @@ -1966,6 +2075,8 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) H5free_memory(astr_val); if(astr_val_exp) HDfree(astr_val_exp); + + /* The reader sends an error message to the writer to stop the test.*/ if(s->use_named_pipes && s->attr_test == true) np_send_error(s,false); @@ -2069,11 +2180,9 @@ static bool verify_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int which) { const char* aname_format = "attr-d-%u-%u"; - //char attrname[sizeof("attr-d-9999999999-999")]; char attrname[VS_ATTR_NAME_LEN]; bool ret = verify_attrs_compact(s,g,max_c,which); if(ret == true) { - //HDsprintf(attrname, aname_format, which,0); HDsprintf(attrname, aname_format, max_c+which,0); ret = vrfy_attr(s,g,which+max_c,attrname,which); if(ret == false) @@ -2086,34 +2195,13 @@ static bool verify_del_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { const char* aname_format = "attr-%u-%u"; - //char attrname[sizeof("attr-d-9999999999-999")]; char attrname[VS_ATTR_NAME_LEN]; bool ret = verify_attrs_compact(s,g,max_c,which); if(ret == true) { - // The writer only deletes the attribute attr-which-0 + /* The writer only deletes the attribute attr-which-0 */ HDsprintf(attrname,aname_format,which,0); - // Add error handling later. - //es = disable_estack(); ret = verify_del_one_attr(s,g,attrname); } -#if 0 - attr_exists = H5Aexists_by_name(g,".",attrname,H5P_DEFAULT); - restore_estack(es); - if(attr_exists == FALSE) { - dbgf(1,"verify_del_attrs_compact() test: \n"); - dbgf(1," attribute %s on group %u is successfully deleted. \n",attrname,which); - ret = true; - } - else if(attr_exists == TRUE) { - dbgf(1,"verify_del_attrs_compact() test failed \n"); - ret = false; - } - else{ - dbgf(1,"H5Aexists_by_name failed \n"); - ret = false; - } - } -#endif return ret; } @@ -2121,32 +2209,12 @@ static bool verify_del_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int which) { const char* aname_format = "attr-d-%u-%u"; - //char attrname[sizeof("attr-d-9999999999-999")]; char attrname[VS_ATTR_NAME_LEN]; bool ret = verify_attrs_compact_dense(s,g,max_c,which); if(ret == true) { - // The writer only deletes the attribute attr-which-0 + /* The writer only deletes the attribute attr-d-which-0 */ HDsprintf(attrname,aname_format,max_c+which,0); - // Add error handling later. ret = verify_del_one_attr(s,g,attrname); -#if 0 - es = disable_estack(); - attr_exists = H5Aexists_by_name(g,".",attrname,H5P_DEFAULT); - restore_estack(es); - if(attr_exists == FALSE) { - dbgf(1,"verify_del_attrs_compact_dense() test: \n"); - dbgf(1," attribute %s on group %u is successfully deleted. \n",attrname,which); - ret = true; - } - else if(attr_exists == TRUE) { - dbgf(1,"verify_del_attrs_compact_dense() test failed \n"); - ret = false; - } - else{ - dbgf(1,"H5Aexists_by_name failed \n"); - ret = false; - } -#endif } return ret; @@ -2160,56 +2228,18 @@ verify_del_attrs_compact_dense_compact(state_t *s, unsigned u; const char* aname_format = "attr-%u-%u"; char attrname[VS_ATTR_NAME_LEN]; - //char attrname[sizeof("attr-9999999999-999")]; + bool ret = verify_attrs_compact_dense(s,g,max_c,which); if(ret == true) { u = max_c + 1; for(u--;u>=(min_d-1);u--) { HDsprintf(attrname, aname_format, which,max_c-u); ret = verify_del_one_attr(s,g,attrname); -#if 0 - es = disable_estack(); - attr_exists = H5Aexists_by_name(g,".",attrname,H5P_DEFAULT); - restore_estack(es); - if(attr_exists == FALSE) { - dbgf(1,"verify_del_attrs_compact_dense_compact() test: \n"); - dbgf(1," attribute %s on group %u is successfully deleted. \n",attrname,which); - ret = true; - } - else if(attr_exists == TRUE) { - dbgf(1,"verify_del_attrs_compact_dense_compact() test failed \n"); - ret = false; - } - else{ - dbgf(1,"H5Aexists_by_name failed \n"); - ret = false; - } -#endif } - // The writer only deletes the attribute attr-which-0 + /* Just verify the one deleted attribute by the writer.*/ HDsprintf(attrname,aname_format,max_c+which,0); ret = verify_del_one_attr(s,g,attrname); - // Add error handling later. - // -#if 0 - es = disable_estack(); - attr_exists = H5Aexists_by_name(g,".",attrname,H5P_DEFAULT); - restore_estack(es); - if(attr_exists == FALSE) { - dbgf(1,"verify_del_attrs_compact_dense() test: \n"); - dbgf(1," attribute %s on group %u is successfully deleted. \n",attrname,which); - ret = true; - } - else if(attr_exists == TRUE) { - dbgf(1,"verify_del_attrs_compact_dense() test failed \n"); - ret = false; - } - else{ - dbgf(1,"H5Aexists_by_name failed \n"); - ret = false; - } -#endif } return ret; @@ -2307,7 +2337,6 @@ verify_group(state_t *s, unsigned int which) hid_t g = H5I_INVALID_HID; bool result = true; - //assert(which < s->nsteps); if (which >= s->nsteps) { H5_FAILED(); AT(); printf("Number of created groups is out of bounds\n"); @@ -2363,981 +2392,185 @@ verify_group(state_t *s, unsigned int which) } -// Temp Name pipe works. -#if 0 -static bool -verify_group(state_t *s, unsigned int which) +int +main(int argc, char **argv) { - char name[sizeof("/group-9999999999")]; - hid_t g; - estack_state_t es; - bool result = true; - bool gopen_ret = true; + hid_t fapl = H5I_INVALID_HID, fcpl = H5I_INVALID_HID; + unsigned step; + bool writer = false; + state_t s; + const char *personality; + H5F_vfd_swmr_config_t config; + const char *fifo_writer_to_reader = "./fifo_group_writer_to_reader"; + const char *fifo_reader_to_writer = "./fifo_group_reader_to_writer"; + int fd_writer_to_reader = -1, fd_reader_to_writer = -1; + int notify = 0, verify = 0; + bool wg_ret = false; + bool vg_ret = false; - assert(which < s->nsteps); + if (!state_init(&s, argc, argv)) { + H5_FAILED(); AT(); + printf("state_init failed\n"); + goto error; + } - esnprintf(name, sizeof(name), "/group-%d", which); + personality = strstr(s.progname, "vfd_swmr_group_"); - if(true == s->attr_test) { - s->np_verify++; - /* Receive the notify that the writer bumped up the value */ - if (HDread(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) - err(EXIT_FAILURE, "read failed"); - if (s->np_notify != s->np_verify) - errx(EXIT_FAILURE, "received message %d, expecting %d", s->np_notify, s->np_verify); - decisleep(3*(s->tick_len)); - dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); - - } - - es = disable_estack(); - g = H5Gopen(s->file, name, H5P_DEFAULT); - restore_estack(es); + if (personality != NULL && + strcmp(personality, "vfd_swmr_group_writer") == 0) + writer = true; + else if (personality != NULL && + strcmp(personality, "vfd_swmr_group_reader") == 0) + writer = false; + else { + H5_FAILED(); AT(); + printf("unknown personality, expected vfd_swmr_group_{reader,writer}\n"); + goto error; + } + + /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ + init_vfd_swmr_config(&config, 4, 7, writer, FALSE, 128, "./group-shadow"); - if (g < 0) - gopen_ret = false; -//if(gopen_ret == true) { -if(s->attr_test == true) { - if(gopen_ret == false) { - dbgf(1, "reader: the gopen verfication failed \n",which); - - s->np_notify = 0; + /* use_latest_format, use_vfd_swmr, only_meta_page, config */ + if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config)) < 0) { + H5_FAILED(); AT(); + printf("vfd_swmr_create_fapl failed\n"); + goto error; } - if (HDwrite(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) - err(EXIT_FAILURE, "write failed"); - dbgf(1, "reader: finish sending back the message: %d\n.", s->np_notify); + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) { + H5_FAILED(); AT(); + printf("H5Pcreate failed\n"); + goto error; } -if(gopen_ret == true) { - if (s->asteps != 0 && which % s->asteps == 0) - result = verify_group_attribute(s, g, which); + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1) < 0) { + H5_FAILED(); AT(); + printf("H5Pset_file_space_strategy failed\n"); + goto error; + } + + if (writer) + s.file = H5Fcreate(s.filename, H5F_ACC_TRUNC, fcpl, fapl); else - result = true; + s.file = H5Fopen(s.filename, H5F_ACC_RDONLY, fapl); + if (s.file < 0) { + H5_FAILED(); AT(); + printf("H5Fcreate/open failed\n"); + goto error; + } -//} + /* Use two named pipes(FIFO) to coordinate the writer and reader for + * two-way communication so that the two sides can move forward together. + * One is for the writer to write to the reader. + * The other one is for the reader to signal the writer. */ + if (s.use_named_pipes && writer) { + /* Writer creates two named pipes(FIFO) */ + if (HDmkfifo(fifo_writer_to_reader, 0600) < 0) { + H5_FAILED(); AT(); + printf("HDmkfifo failed\n"); + goto error; + } - if (H5Gclose(g) < 0) - errx(EXIT_FAILURE, "H5Gclose failed"); -} -else - result = false; + if (HDmkfifo(fifo_reader_to_writer, 0600) < 0) { + H5_FAILED(); AT(); + printf("HDmkfifo failed\n"); + goto error; + } - return result; -} + } -#endif + /* Both the writer and reader open the pipes */ + if (s.use_named_pipes && (fd_writer_to_reader = HDopen(fifo_writer_to_reader, O_RDWR)) < 0) { + H5_FAILED(); AT(); + printf("HDopen failed\n"); + goto error; + } + if (s.use_named_pipes && (fd_reader_to_writer = HDopen(fifo_reader_to_writer, O_RDWR)) < 0) { + H5_FAILED(); AT(); + printf("HDopen failed\n"); + goto error; + } -#if 0 -static bool -add_group_attribute(state_t *s, hid_t g, hid_t sid, unsigned int which) -{ - hid_t aid; - char name[sizeof("attr-9999999999")]; + if(s.use_named_pipes) { + s.np_fd_w_to_r = fd_writer_to_reader; + s.np_fd_r_to_w = fd_reader_to_writer; + s.np_notify = notify; + s.np_verify = verify; + s.tick_len = config.tick_len; + s.max_lag = config.max_lag; + } - esnprintf(name, sizeof(name), "attr-%u", which); + /* For attribute test, force the named pipe to communicate for every step. */ + if (s.at_pattern != ' ') { + s.attr_test = true; + if(s.use_named_pipes) + s.csteps = 1; + } + if (writer) { + for (step = 0; step < s.nsteps; step++) { + dbgf(2, "writer: step %d\n", step); - dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); + wg_ret = write_group(&s, step); - if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, - H5P_DEFAULT)) < 0) - errx(EXIT_FAILURE, "H5Acreate2 failed"); + if(wg_ret == false) { + H5_FAILED(); AT(); + printf("write_group failed at step %d\n",step); - if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) - errx(EXIT_FAILURE, "H5Awrite failed"); - if (H5Aclose(aid) < 0) - errx(EXIT_FAILURE, "H5Aclose failed"); -} + /* At communication interval, notifies the reader about the failture and quit */ + if (s.use_named_pipes && s.attr_test !=true && step % s.csteps == 0) + np_send_error(&s,true); + goto error; + } + else { + /* At communication interval, notifies the reader and waits for its response */ + if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { -static bool -write_group(state_t *s, unsigned int which) -{ - char name[sizeof("/group-9999999999")]; - hid_t g; + if(np_wr_send_receive(&s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write group - verification failed.\n"); + goto error; + } + } + } + } + } else { + for (step = 0; step < s.nsteps;step++) { + dbgf(1, "reader: step %d\n", step); - assert(which < s->nsteps); + /* At communication interval, waits for the writer to finish creation before starting verification */ + if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { + if(false == np_rd_receive(&s)) { + H5_FAILED(); AT(); + goto error; + } + } - esnprintf(name, sizeof(name), "/group-%d", which); + /* For the default test, wait for a few ticks for the update to happen */ + if(s.use_named_pipes && s.attr_test== false) + decisleep(config.tick_len * s.update_interval); - g = H5Gcreate2(s->file, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + vg_ret = verify_group(&s, step); - if (g < 0) - errx(EXIT_FAILURE, "H5Gcreate(, \"%s\", ) failed", name); + if (vg_ret == false) { - if (s->asteps != 0 && which % s->asteps == 0) - add_group_attribute(s, g, s->one_by_one_sid, which); + printf("verify_group failed\n"); + H5_FAILED(); AT(); - if (H5Gclose(g) < 0) - errx(EXIT_FAILURE, "H5Gclose failed"); -} - -static bool -verify_group_attribute(hid_t g, unsigned int which) -{ - estack_state_t es; - unsigned int read_which; - hid_t aid; - char name[sizeof("attr-9999999999")]; - - esnprintf(name, sizeof(name), "attr-%u", which); - - dbgf(1, "verifying attribute %s on group %u equals %u\n", name, which, - which); - - es = disable_estack(); - if ((aid = H5Aopen(g, name, H5P_DEFAULT)) < 0) { - restore_estack(es); - return false; - } - - if (H5Aread(aid, H5T_NATIVE_UINT, &read_which) < 0) { - restore_estack(es); - if (H5Aclose(aid) < 0) - errx(EXIT_FAILURE, "H5Aclose failed"); - return false; - } - - restore_estack(es); - - if (H5Aclose(aid) < 0) - errx(EXIT_FAILURE, "H5Aclose failed"); - - return read_which == which; -} - -static bool -verify_group(state_t *s, unsigned int which) -{ - char name[sizeof("/group-9999999999")]; - hid_t g; - estack_state_t es; - bool result; - - assert(which < s->nsteps); - - esnprintf(name, sizeof(name), "/group-%d", which); - - es = disable_estack(); - g = H5Gopen(s->file, name, H5P_DEFAULT); - restore_estack(es); - - if (g < 0) - return false; - - if (s->asteps != 0 && which % s->asteps == 0) - result = verify_group_attribute(g, which); - else - result = true; - - if (H5Gclose(g) < 0) - errx(EXIT_FAILURE, "H5Gclose failed"); - - return result; -} - -#endif - -//OLDWORK -#if 0 -static bool -add_group_attribute(const state_t *s, hid_t g, hid_t sid, unsigned int which) -{ - hid_t aid; - char name[sizeof("attr-9999999999")]; - - esnprintf(name, sizeof(name), "attr-%u", which); - - dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); - - if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, - H5P_DEFAULT)) < 0) { - H5_FAILED(); AT(); - printf("H5Acreate2 failed\n"); - goto error; - } - - if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) { - H5_FAILED(); AT(); - printf("H5Awrite failed\n"); - goto error; - } - - if (H5Aclose(aid) < 0) { - H5_FAILED(); AT(); - printf("H5Aclose failed\n"); - goto error; - } - - return true; - -error: - H5E_BEGIN_TRY { - H5Aclose(aid); - } H5E_END_TRY; - - return false; -} - -static bool -write_group(state_t *s, unsigned int which) -{ - char name[sizeof("/group-9999999999")]; - hid_t g = H5I_INVALID_HID; - bool result = true; - - if (which >= s->nsteps) { - H5_FAILED(); AT(); - printf("group order is out of bounds\n"); - goto error; - } - - esnprintf(name, sizeof(name), "/group-%d", which); - - if ((g = H5Gcreate2(s->file, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) { - H5_FAILED(); AT(); - printf("H5Gcreate2 failed\n"); - goto error; - } - - if (s->asteps != 0 && which % s->asteps == 0) - result = add_group_attribute(s, g, s->one_by_one_sid, which); - - if (H5Gclose(g) < 0) { - H5_FAILED(); AT(); - printf("H5Gclose failed\n"); - goto error; - } - - return result; - -error: - H5E_BEGIN_TRY { - H5Gclose(g); - } H5E_END_TRY; - - return false; -} - -static bool -verify_group_attribute(hid_t g, unsigned int which) -{ - unsigned int read_which; - hid_t aid; - char name[sizeof("attr-9999999999")]; - - esnprintf(name, sizeof(name), "attr-%u", which); - - dbgf(1, "verifying attribute %s on group %u equals %u\n", name, which, which); - - if ((aid = H5Aopen(g, name, H5P_DEFAULT)) < 0) { - H5_FAILED(); AT(); - printf("H5Aopen failed\n"); - goto error; - } - - if (H5Aread(aid, H5T_NATIVE_UINT, &read_which) < 0) { - H5_FAILED(); AT(); - printf("H5Aread failed\n"); - goto error; - } - - if (read_which != which) { - H5_FAILED(); AT(); - printf("H5Aread wrong value\n"); - goto error; - } - - if (H5Aclose(aid) < 0) { - H5_FAILED(); AT(); - printf("H5Aread failed\n"); - goto error; - } - - return true; - -error: - H5E_BEGIN_TRY { - H5Aclose(aid); - } H5E_END_TRY; - - return false; -} - -static bool -verify_group(state_t *s, unsigned int which) -{ - char name[sizeof("/group-9999999999")]; - hid_t g = H5I_INVALID_HID; - bool result = true; - - if (which >= s->nsteps) { - H5_FAILED(); AT(); - printf("Group order is out of bounds\n"); - goto error; - } - - esnprintf(name, sizeof(name), "/group-%d", which); - - if ((g = H5Gopen(s->file, name, H5P_DEFAULT)) < 0) { - H5_FAILED(); AT(); - printf("H5Gopen failed\n"); - goto error; - } - - if (s->asteps != 0 && which % s->asteps == 0) - result = verify_group_attribute(g, which); - else - result = true; - - if (H5Gclose(g) < 0) { - H5_FAILED(); AT(); - printf("H5Gclose failed\n"); - goto error; - } - - return result; - -error: - H5E_BEGIN_TRY { - H5Gclose(g); - } H5E_END_TRY; - - return false; -} - -int -main(int argc, char **argv) -{ - hid_t fapl = H5I_INVALID_HID, fcpl = H5I_INVALID_HID; - unsigned step; - bool writer = false; - state_t s; - const char *personality; - H5F_vfd_swmr_config_t config; - const char *fifo_writer_to_reader = "./fifo_group_writer_to_reader"; - const char *fifo_reader_to_writer = "./fifo_group_reader_to_writer"; - int fd_writer_to_reader = -1, fd_reader_to_writer = -1; - int notify = 0, verify = 0; - unsigned int i; - - if (!state_init(&s, argc, argv)) { - H5_FAILED(); AT(); - printf("state_init failed\n"); - goto error; - } - - personality = strstr(s.progname, "vfd_swmr_group_"); - - if (personality != NULL && strcmp(personality, "vfd_swmr_group_writer") == 0) - writer = true; - else if (personality != NULL && strcmp(personality, "vfd_swmr_group_reader") == 0) - writer = false; - else { - H5_FAILED(); AT(); - printf("unknown personality, expected vfd_swmr_group_{reader,writer}\n"); - goto error; - } - - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, 4, 7, writer, FALSE, 128, "./group-shadow"); - - /* use_latest_format, use_vfd_swmr, only_meta_page, config */ - if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config)) < 0) { - H5_FAILED(); AT(); - printf("vfd_swmr_create_fapl failed\n"); - goto error; - } - - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) { - H5_FAILED(); AT(); - printf("H5Pcreate failed\n"); - goto error; - } - - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1) < 0) { - H5_FAILED(); AT(); - printf("H5Pset_file_space_strategy failed\n"); - goto error; - } - - if (writer) - s.file = H5Fcreate(s.filename, H5F_ACC_TRUNC, fcpl, fapl); - else - s.file = H5Fopen(s.filename, H5F_ACC_RDONLY, fapl); - - if (s.file < 0) { - H5_FAILED(); AT(); - printf("H5Fcreate/open failed\n"); - goto error; - } - - /* Use two named pipes(FIFO) to coordinate the writer and reader for - * two-way communication so that the two sides can move forward together. - * One is for the writer to write to the reader. - * The other one is for the reader to signal the writer. */ - if (s.use_named_pipes && writer) { - /* Writer creates two named pipes(FIFO) */ - if (HDmkfifo(fifo_writer_to_reader, 0600) < 0) { - H5_FAILED(); AT(); - printf("HDmkfifo failed\n"); - goto error; - } - - if (HDmkfifo(fifo_reader_to_writer, 0600) < 0) { - H5_FAILED(); AT(); - printf("HDmkfifo failed\n"); - goto error; - } - - } - - /* Both the writer and reader open the pipes */ - if (s.use_named_pipes && (fd_writer_to_reader = HDopen(fifo_writer_to_reader, O_RDWR)) < 0) { - H5_FAILED(); AT(); - printf("HDopen failed\n"); - goto error; - } - - if (s.use_named_pipes && (fd_reader_to_writer = HDopen(fifo_reader_to_writer, O_RDWR)) < 0) { - H5_FAILED(); AT(); - printf("HDopen failed\n"); - goto error; - } - - if (writer) { - for (step = 0; step < s.nsteps; step++) { - dbgf(2, "writer: step %d\n", step); - - if (!write_group(&s, step)) { - H5_FAILED(); AT(); - printf("write_group failed\n"); - - /* At communication interval, notifies the reader about the failture and quit */ - if (s.use_named_pipes && (step % s.csteps == 0)) { - notify = -1; - HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)); - } - - goto error; - } else { - /* At communication interval, notifies the reader and waits for its response */ - if (s.use_named_pipes && (step % s.csteps == 0)) { - /* Bump up the value of notify to notice the reader to start to read */ - notify++; - if (HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDwrite failed\n"); - goto error; - } - - /* During the wait, writer makes repeated HDF5 API calls - * to trigger EOT at approximately the correct time */ - for(i = 0; i < config.max_lag + 1; i++) { - decisleep(config.tick_len); - H5E_BEGIN_TRY { - H5Aexists(s.file, "nonexistent"); - } H5E_END_TRY; - } - - /* Receive the same value from the reader and verify it before - * going to the next step */ - verify++; - if (HDread(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDread failed\n"); - goto error; - } - - if (notify == -1) { - H5_FAILED(); AT(); - printf("reader failed to verify group\n"); - goto error; - } - - if (notify != verify) { - H5_FAILED(); AT(); - printf("received message %d, expecting %d\n", notify, verify); - goto error; - } - } - } - } - } - else { - for (step = 0; step < s.nsteps; step++) { - dbgf(2, "reader: step %d\n", step); - - /* At communication interval, waits for the writer to finish creation before starting verification - */ - if (s.use_named_pipes && (step % s.csteps == 0)) { - /* The writer should have bumped up the value of notify. - * Do the same with verify and confirm it */ - verify++; - - /* Receive the notify that the writer bumped up the value */ - if (HDread(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDread failed\n"); - goto error; - } - - if (notify == -1) { - H5_FAILED(); AT(); - printf("writer failed to create group\n"); - goto error; - } - - if (notify != verify) { - H5_FAILED(); AT(); - printf("received message %d, expecting %d\n", notify, verify); - goto error; - } - } - - /* Wait for a few ticks for the update to happen */ - if (s.use_named_pipes) - decisleep(config.tick_len * s.update_interval); - - /* Start to verify group */ - if (!verify_group(&s, step)) { - H5_FAILED(); AT(); - printf("verify_group failed\n"); - - /* At communication interval, tell the writer about the failure and exit */ - if (s.use_named_pipes && (step % s.csteps == 0)) { - notify = -1; - HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)); - } - - goto error; - } else { - if (s.use_named_pipes && (step % s.csteps == 0)) { - /* Send back the same nofity value for acknowledgement to tell the writer - * move to the next step */ - if (HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDwrite failed\n"); - goto error; - } - } - } - } - } - - if (H5Pclose(fapl) < 0) { - H5_FAILED(); AT(); - printf("H5Pclose failed\n"); - goto error; - } - - if (H5Pclose(fcpl) < 0) { - H5_FAILED(); AT(); - printf("H5Pclose failed\n"); - goto error; - } - - if (H5Fclose(s.file) < 0) { - H5_FAILED(); AT(); - printf("H5Fclose failed\n"); - goto error; - } - - /* Both the writer and reader close the named pipes */ - if (s.use_named_pipes && HDclose(fd_writer_to_reader) < 0) { - H5_FAILED(); AT(); - printf("HDclose failed\n"); - goto error; - } - - if (s.use_named_pipes && HDclose(fd_reader_to_writer) < 0) { - H5_FAILED(); AT(); - printf("HDclose failed\n"); - goto error; - } - - /* Reader finishes last and deletes the named pipes */ - if(s.use_named_pipes && !writer) { - if(HDremove(fifo_writer_to_reader) != 0) { - H5_FAILED(); AT(); - printf("HDremove failed\n"); - goto error; - } - - if(HDremove(fifo_reader_to_writer) != 0) { - H5_FAILED(); AT(); - printf("HDremove failed\n"); - goto error; - } - } - - return EXIT_SUCCESS; - -error: - H5E_BEGIN_TRY { - H5Pclose(fapl); - H5Pclose(fcpl); - H5Fclose(s.file); - } H5E_END_TRY; - - if (s.use_named_pipes && fd_writer_to_reader >= 0) - HDclose(fd_writer_to_reader); - - if (s.use_named_pipes && fd_reader_to_writer >= 0) - HDclose(fd_reader_to_writer); - - if(s.use_named_pipes && !writer) { - HDremove(fifo_writer_to_reader); - HDremove(fifo_reader_to_writer); - } - - return EXIT_FAILURE; -} -#endif - -int -main(int argc, char **argv) -{ - -#if 0 - hid_t fapl, fcpl; - herr_t ret; - unsigned step; - bool writer; - state_t s; - const char *personality; - H5F_vfd_swmr_config_t config; - const char *fifo_writer_to_reader = "./fifo_group_writer_to_reader"; - const char *fifo_reader_to_writer = "./fifo_group_reader_to_writer"; - int fd_writer_to_reader, fd_reader_to_writer; - // notify = 0 and verify = 0 are for error. - int notify = 1, verify = 1; - unsigned int i; - - state_init(&s, argc, argv); - - personality = strstr(s.progname, "vfd_swmr_group_"); - - if (personality != NULL && - strcmp(personality, "vfd_swmr_group_writer") == 0) - writer = true; - else if (personality != NULL && - strcmp(personality, "vfd_swmr_group_reader") == 0) - writer = false; - else { - errx(EXIT_FAILURE, - "unknown personality, expected vfd_swmr_group_{reader,writer}"); - } - - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, 4, 7, writer, FALSE, 128, "./group-shadow"); - - /* use_latest_format, use_vfd_swmr, only_meta_page, config */ - fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config); - - if (fapl < 0) - errx(EXIT_FAILURE, "vfd_swmr_create_fapl"); - - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - errx(EXIT_FAILURE, "H5Pcreate"); - - ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1); - if (ret < 0) - errx(EXIT_FAILURE, "H5Pset_file_space_strategy"); - - if (writer) - s.file = H5Fcreate(s.filename, H5F_ACC_TRUNC, fcpl, fapl); - else - s.file = H5Fopen(s.filename, H5F_ACC_RDONLY, fapl); - - if (s.file == badhid) - errx(EXIT_FAILURE, writer ? "H5Fcreate" : "H5Fopen"); - - /* Use two named pipes(FIFO) to coordinate the writer and reader for - * two-way communication so that the two sides can move forward together. - * One is for the writer to write to the reader. - * The other one is for the reader to signal the writer. */ - if (writer) { - /* Writer creates two named pipes(FIFO) */ - if (HDmkfifo(fifo_writer_to_reader, 0600) < 0) - errx(EXIT_FAILURE, "HDmkfifo"); - - if (HDmkfifo(fifo_reader_to_writer, 0600) < 0) - errx(EXIT_FAILURE, "HDmkfifo"); - } - - /* Both the writer and reader open the pipes */ - if ((fd_writer_to_reader = HDopen(fifo_writer_to_reader, O_RDWR)) < 0) - errx(EXIT_FAILURE, "fifo_writer_to_reader open failed"); - - if ((fd_reader_to_writer = HDopen(fifo_reader_to_writer, O_RDWR)) < 0) - errx(EXIT_FAILURE, "fifo_reader_to_writer open failed"); - -#endif - - hid_t fapl = H5I_INVALID_HID, fcpl = H5I_INVALID_HID; - unsigned step; - bool writer = false; - state_t s; - const char *personality; - H5F_vfd_swmr_config_t config; - const char *fifo_writer_to_reader = "./fifo_group_writer_to_reader"; - const char *fifo_reader_to_writer = "./fifo_group_reader_to_writer"; - int fd_writer_to_reader = -1, fd_reader_to_writer = -1; - int notify = 0, verify = 0; - unsigned int i; - bool wg_ret = false; - bool vg_ret = false; - - if (!state_init(&s, argc, argv)) { - H5_FAILED(); AT(); - printf("state_init failed\n"); - goto error; - } - - personality = strstr(s.progname, "vfd_swmr_group_"); - - if (personality != NULL && - strcmp(personality, "vfd_swmr_group_writer") == 0) - writer = true; - else if (personality != NULL && - strcmp(personality, "vfd_swmr_group_reader") == 0) - writer = false; - else { - H5_FAILED(); AT(); - printf("unknown personality, expected vfd_swmr_group_{reader,writer}\n"); - goto error; - } - - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, 4, 7, writer, FALSE, 128, "./group-shadow"); - - /* use_latest_format, use_vfd_swmr, only_meta_page, config */ - if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config)) < 0) { - H5_FAILED(); AT(); - printf("vfd_swmr_create_fapl failed\n"); - goto error; - } - - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) { - H5_FAILED(); AT(); - printf("H5Pcreate failed\n"); - goto error; - } - - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1) < 0) { - H5_FAILED(); AT(); - printf("H5Pset_file_space_strategy failed\n"); - goto error; - } - - if (writer) - s.file = H5Fcreate(s.filename, H5F_ACC_TRUNC, fcpl, fapl); - else - s.file = H5Fopen(s.filename, H5F_ACC_RDONLY, fapl); - - if (s.file < 0) { - H5_FAILED(); AT(); - printf("H5Fcreate/open failed\n"); - goto error; - } - - /* Use two named pipes(FIFO) to coordinate the writer and reader for - * two-way communication so that the two sides can move forward together. - * One is for the writer to write to the reader. - * The other one is for the reader to signal the writer. */ - if (s.use_named_pipes && writer) { - /* Writer creates two named pipes(FIFO) */ - if (HDmkfifo(fifo_writer_to_reader, 0600) < 0) { - H5_FAILED(); AT(); - printf("HDmkfifo failed\n"); - goto error; - } - - if (HDmkfifo(fifo_reader_to_writer, 0600) < 0) { - H5_FAILED(); AT(); - printf("HDmkfifo failed\n"); - goto error; - } - - } - - /* Both the writer and reader open the pipes */ - if (s.use_named_pipes && (fd_writer_to_reader = HDopen(fifo_writer_to_reader, O_RDWR)) < 0) { - H5_FAILED(); AT(); - printf("HDopen failed\n"); - goto error; - } - - if (s.use_named_pipes && (fd_reader_to_writer = HDopen(fifo_reader_to_writer, O_RDWR)) < 0) { - H5_FAILED(); AT(); - printf("HDopen failed\n"); - goto error; - } - - if(s.use_named_pipes) { - s.np_fd_w_to_r = fd_writer_to_reader; - s.np_fd_r_to_w = fd_reader_to_writer; - s.np_notify = notify; - s.np_verify = verify; - s.tick_len = config.tick_len; - s.max_lag = config.max_lag; - } - - // TODO: use a different name for s pointer(sp?) in subroutine, - // TODO: since its name is also defined as s. - - // For attribute test, force the named pipe to communicate for every step. - if (s.at_pattern != ' ') { - s.attr_test = true; - if(s.use_named_pipes) - s.csteps = 1; - } - if (writer) { - for (step = 0; step < s.nsteps; step++) { - dbgf(2, "writer: step %d\n", step); - - wg_ret = write_group(&s, step); - - if(wg_ret == false) { - H5_FAILED(); AT(); - printf("write_group failed at step %d\n",step); - - /* At communication interval, notifies the reader about the failture and quit */ - if (s.use_named_pipes && s.attr_test !=true && step % s.csteps == 0) { - //if(1){ - #if 0 - s.np_notify = -1; - HDwrite(fd_writer_to_reader, &(s.np_notify), sizeof(int)); - #endif - np_send_error(&s,true); - } - goto error; - } - else { - - /* At communication interval, notifies the reader and waits for its response */ - if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { - - if(np_wr_send_receive(&s) == false) { - H5_FAILED(); AT(); - dbgf(2, "writer: write group - verification failed.\n"); - goto error; - } -#if 0 - /* Bump up the value of notify to notice the reader to start to read */ - s.np_notify++; - if (HDwrite(fd_writer_to_reader, &(s.np_notify), sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDwrite failed\n"); - goto error; - } - - /* During the wait, writer makes repeated HDF5 API calls - * to trigger EOT at approximately the correct time */ - for(i = 0; i < config.max_lag + 1; i++) { - decisleep(config.tick_len); - H5E_BEGIN_TRY { - H5Aexists(s.file, "nonexistent"); - } H5E_END_TRY; - } - - /* Receive the same value from the reader and verify it before - * going to the next step */ - (s.np_verify)++; - if (HDread(fd_reader_to_writer, &(s.np_notify), sizeof(int)) < 0){ - H5_FAILED(); AT(); - printf("HDread failed\n"); - goto error; - } - - if (s.np_notify == -1) { - H5_FAILED(); AT(); - printf("reader failed to verify group\n"); - goto error; - } - - if (s.np_notify != s.np_verify) { - H5_FAILED(); AT(); - printf("received message %d, expecting %d\n", s.np_notify, s.np_verify); - goto error; - } -#endif - } - } - } - } else { - for (step = 0; step < s.nsteps;step++) { - dbgf(1, "reader: step %d\n", step); - - /* At communication interval, waits for the writer to finish creation before starting verification */ - if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { - if(false == np_rd_receive(&s)) { - H5_FAILED(); AT(); - goto error; - } -#if 0 - /* The writer should have bumped up the value of notify. - * Do the same with verify and confirm it */ - s.np_verify++; - - /* Receive the notify that the writer bumped up the value */ - if (HDread(fd_writer_to_reader, &(s.np_notify), sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDread failed\n"); - goto error; - } - - if (s.np_notify == -1) { - H5_FAILED(); AT(); - printf("writer failed to create group\n"); - goto error; - } - - if (s.np_notify != s.np_verify) { - H5_FAILED(); AT(); - printf("received message %d, expecting %d\n", s.np_notify, s.np_verify); - goto error; - } -#endif - } - - /* For the default test, wait for a few ticks for the update to happen */ - if(s.use_named_pipes && s.attr_test== false) - decisleep(config.tick_len * s.update_interval); - - vg_ret = verify_group(&s, step); - - if (vg_ret == false) { - - printf("verify_group failed\n"); - H5_FAILED(); AT(); - /* At communication interval, tell the writer about the failure and exit */ - if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { - //if(1){ - np_send_error(&s,false); - //s.np_notify = -1; - //HDwrite(fd_reader_to_writer, &(s.np_notify), sizeof(int)); - } - goto error; + /* At communication interval, tell the writer about the failure and exit */ + if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) + np_send_error(&s,false); + goto error; } else { /* Send back the same nofity value for acknowledgement to tell the writer * move to the next step. */ - // TO THINK: reader will never have a chance to acknowledge the writer when attribute verfication occurs. - // RESOLVED, no need to carry out the following for the attribute operation. It is done in the attribute level. if (s.use_named_pipes && s.attr_test!=true && step % s.csteps == 0) { -#if 0 - if (HDwrite(fd_reader_to_writer, &(s.np_notify), sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDwrite failed\n"); - goto error; - } -#endif if(np_rd_send(&s)==false) goto error; } From 90cbdd34c849689caf430570042356f0981ac4c3 Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Fri, 23 Apr 2021 12:11:59 -0500 Subject: [PATCH 06/13] correct the code a bit, add comments to the shell script. --- test/testvfdswmr.sh.in | 12 +- test/vfd_swmr_group_writer.c | 395 +++++++++++++++++++++++++++++++++-- 2 files changed, 388 insertions(+), 19 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index 1a1d6837bb7..9dc61ee325f 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -654,6 +654,16 @@ if [ ${do_groups:-no} = yes ]; then rm -f vfd_swmr_group_reader.*.{out,rc} fi +# The group attribute test takes longer. +# So for standard run and quick run, we +# shorten the number of tests. The standard +# run covers all the features we need to +# test. The quick run doesn't cover the +# attribute storage change between dense and +# compact. +# The exhaustive run tries to test a feature +# per test from scratch. +# grp_attr_list=( "compact" "dense" @@ -675,13 +685,11 @@ grp_sub_attr_list=( grp_short_sub_attr_list=( "dense" - "compact-del" "modify" "remove-vstr" "modify-vstr" ) -echo "$HDF5TestExpress" if [[ "$HDF5TestExpress" -eq 1 ]] ; then #Setting for standard run grp_attr_list=("${grp_sub_attr_list[@]}") elif [[ "$HDF5TestExpress" -gt 1 ]] ; then #Setting for quick run diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 4627dc69a7a..7ffb0488cfe 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -2086,11 +2086,39 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) } +/*------------------------------------------------------------------------- + * Function: verify_del_one_attr + * + * Purpose: verify if an attribute is successfully deleted. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * const char* aname + * The name of the attribute to be deleted. + * + * Return: Success: true + * Failure: false + * + * Note: This is an internal function used by "remove-vlstr", + * "compact-del","dense-del",dense-del-to-compact" tests. + *------------------------------------------------------------------------- +*/ + + static bool verify_del_one_attr(state_t *s,hid_t g, const char *aname) { htri_t attr_exists = FALSE; + /* The reader receives a message from the writer.Then sleep + * for a few ticks or stop the test if the received message + * is an error message. + */ if(s->use_named_pipes && true == s->attr_test) { if(false == np_rd_receive(s)) { H5_FAILED(); AT(); @@ -2100,6 +2128,7 @@ verify_del_one_attr(state_t *s,hid_t g, const char *aname) { dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); } + /* Check if the deleted attribute still exists. */ attr_exists = H5Aexists_by_name(g,".",aname,H5P_DEFAULT); if(attr_exists == FALSE) { dbgf(1,"verify_del_attrs_compact() test: \n"); @@ -2114,6 +2143,7 @@ verify_del_one_attr(state_t *s,hid_t g, const char *aname) { goto error; } + /* Reader sends an OK message back to the reader */ if(s->use_named_pipes && s->attr_test == true) { if(np_rd_send(s)==false) goto error; @@ -2122,6 +2152,7 @@ verify_del_one_attr(state_t *s,hid_t g, const char *aname) { return true; error: + /* The reader sends an error message to the writer to stop the test.*/ if(s->use_named_pipes && s->attr_test == true) np_send_error(s,false); @@ -2129,6 +2160,35 @@ verify_del_one_attr(state_t *s,hid_t g, const char *aname) { return false; } +/*------------------------------------------------------------------------- + * Function: verify_remove_vlstr_attr + * + * Purpose: Verify if an variable length string attribute is + * successfully deleted by the writer. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * the attribute name. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is for the "remove-vstr" test. + * Also note this function first verifies if + * a variable length attribute is added then + * it verifies if it is deleted successfully. + *------------------------------------------------------------------------- +*/ + static bool verify_remove_vlstr_attr(state_t* s,hid_t g, unsigned int which) { @@ -2144,6 +2204,35 @@ verify_remove_vlstr_attr(state_t* s,hid_t g, unsigned int which) return ret; } +/*------------------------------------------------------------------------- + * Function: verify_modify_vlstr_attr + * + * Purpose: Verify if an variable length string attribute is + * successfully modified by the writer. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * the attribute name. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is for the "modify-vstr" test. + * Also note this function first verifies if + * a variable length attribute is added then + * it verifies if it is modified successfully. + *------------------------------------------------------------------------- +*/ + static bool verify_modify_vlstr_attr(state_t *s, hid_t g, unsigned int which){ @@ -2156,6 +2245,36 @@ verify_modify_vlstr_attr(state_t *s, hid_t g, unsigned int which){ } +/*------------------------------------------------------------------------- + * Function: verify_attrs_compact + * + * Purpose: verify if attributes are successfully added for the compact + * storage. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigend max_c + * The maximal number of attributes the compact storage + * can hold + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct the + * attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used by the "compact" test. + *------------------------------------------------------------------------- +*/ + static bool verify_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { @@ -2163,6 +2282,8 @@ verify_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { bool ret = true; const char* aname_format = "attr-%u-%u"; char attrname[VS_ATTR_NAME_LEN]; + + /* Need to verify the added attribute one by one. */ for (u = 0; u < max_c; u++) { HDsprintf(attrname, aname_format, which,u); @@ -2176,75 +2297,267 @@ verify_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { } +/*------------------------------------------------------------------------- + * Function: verify_attrs_compact_dense + * + * Purpose: verify if attributes are successfully added first in the + * compact storage then in the dense storage. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigend max_c + * The maximal number of attributes the compact storage + * can hold + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used by the "compact-dense" test. + *------------------------------------------------------------------------- +*/ + + static bool verify_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int which) { const char* aname_format = "attr-d-%u-%u"; char attrname[VS_ATTR_NAME_LEN]; + bool ret = verify_attrs_compact(s,g,max_c,which); + if(ret == true) { + + /* Now the storage is in dense. Verify if the + * retrieved value is correct. */ HDsprintf(attrname, aname_format, max_c+which,0); ret = vrfy_attr(s,g,which+max_c,attrname,which); if(ret == false) dbgf(1,"verify_attrs_compact_dense failed \n"); + } return ret; } +/*------------------------------------------------------------------------- + * Function: verify_del_attrs_compact + * + * Purpose: verify if an attribute in compact storage is successfully + * deleted. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigend max_c + * The maximal number of attributes the compact storage + * can hold + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used by the "compact-del" test. + * Also note this function first verifies if + * attributes are successfully added in compact storage then + * it verifies if one added attribute is deleted successfully. + *------------------------------------------------------------------------- +*/ + static bool verify_del_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { const char* aname_format = "attr-%u-%u"; char attrname[VS_ATTR_NAME_LEN]; + bool ret = verify_attrs_compact(s,g,max_c,which); + if(ret == true) { /* The writer only deletes the attribute attr-which-0 */ HDsprintf(attrname,aname_format,which,0); ret = verify_del_one_attr(s,g,attrname); } + return ret; } +/*------------------------------------------------------------------------- + * Function: verify_del_attrs_compact_dense + * + * Purpose: verify if an attribute in dense storage is successfully + * deleted. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigend max_c + * The maximal number of attributes the compact storage + * can hold + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used by the "dense-del" test. + * Also note this function first verifies if + * attributes are successfully added in compact storage then + * in dense storage. Afterwards, + * it verifies if one added attribute is deleted successfully. + *------------------------------------------------------------------------- +*/ + + static bool -verify_del_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int which) { +verify_del_attrs_compact_dense(state_t *s, + hid_t g, + unsigned max_c, + unsigned int which) { const char* aname_format = "attr-d-%u-%u"; char attrname[VS_ATTR_NAME_LEN]; + bool ret = verify_attrs_compact_dense(s,g,max_c,which); + if(ret == true) { /* The writer only deletes the attribute attr-d-which-0 */ HDsprintf(attrname,aname_format,max_c+which,0); ret = verify_del_one_attr(s,g,attrname); } + return ret; } + +/*------------------------------------------------------------------------- + * Function: verify_del_attrs_compact_dense_compact + * + * Purpose: verify that the attributes are deleted successfully + * even the attribute storage changes from dense to + * compact. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigend max_c + * The maximal number of attributes the compact storage + * can hold + * + * unsigend min_d + * The minimal number of attributes to be stored in + * dense storage + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used by the "dense-del-to-compact" test. + * Also note this function first verifies if + * attributes are successfully added in compact storage then + * in dense storage. Afterwards, + * it verifies if some added attributes are deleted successfully + * until the storage changes from dense to compact. + *------------------------------------------------------------------------- +*/ + + static bool verify_del_attrs_compact_dense_compact(state_t *s, hid_t g, unsigned max_c, unsigned min_d, unsigned int which) { + unsigned u; const char* aname_format = "attr-%u-%u"; char attrname[VS_ATTR_NAME_LEN]; + /* Verify the attributes are added correctly from + * compact to dense storage*/ bool ret = verify_attrs_compact_dense(s,g,max_c,which); + if(ret == true) { + + /* Then verify the deletion of attributes + * from dense to compact. + */ u = max_c + 1; for(u--;u>=(min_d-1);u--) { HDsprintf(attrname, aname_format, which,max_c-u); ret = verify_del_one_attr(s,g,attrname); } - /* Just verify the one deleted attribute by the writer.*/ + /* Just verify one more deleted attribute by the writer. + The storage is still compact. */ HDsprintf(attrname,aname_format,max_c+which,0); ret = verify_del_one_attr(s,g,attrname); } + return ret; } +/*------------------------------------------------------------------------- + * Function: verify_group_attribute + * + * Purpose: Check the attribute test pattern and then call the + * correponding verification function. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * group and attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This is called by the verify_group() function. + *------------------------------------------------------------------------- +*/ + + static bool verify_group_attribute(state_t *s, hid_t g, unsigned int which) { @@ -2253,6 +2566,12 @@ verify_group_attribute(state_t *s, hid_t g, unsigned int which) unsigned max_compact = 0; unsigned min_dense = 0; hid_t gcpl = H5I_INVALID_HID; + + /* For tests "compact","compact-to-dense","compact-del", + * "dense-del", "dense-del-to-compact", + * the maximal number of attributes for the compact storage + * and the minimal number of attributes for the dense storage + * are needed. So obtain them here */ switch (test_pattern) { case 'c': case 't': @@ -2284,6 +2603,8 @@ verify_group_attribute(state_t *s, hid_t g, unsigned int which) default: break; } + + /* Distribute the verification test. */ switch (test_pattern) { case 'c': ret = verify_attrs_compact(s, g, max_compact, which); @@ -2330,6 +2651,30 @@ verify_group_attribute(state_t *s, hid_t g, unsigned int which) return false; } +/*------------------------------------------------------------------------- + * Function: verify_group + * + * Purpose: verify the success of group creation and + * carry out the test for attribute operations(add,delete etc.) + * according to the attribute test pattern. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * unsigned int which + * The number of iterations for group creation + * + * + * Return: Success: true + * Failure: false + * + * Note: This is called by the main() function. + *------------------------------------------------------------------------- +*/ + + + static bool verify_group(state_t *s, unsigned int which) { @@ -2337,40 +2682,46 @@ verify_group(state_t *s, unsigned int which) hid_t g = H5I_INVALID_HID; bool result = true; - if (which >= s->nsteps) { - H5_FAILED(); AT(); - printf("Number of created groups is out of bounds\n"); - goto error; - } - - esnprintf(name, sizeof(name), "/group-%d", which); - + /* The reader receives a message from the writer.Then sleep + * for a few ticks or stop the test if the received message + * is an error message. + */ if(s->use_named_pipes && true == s->attr_test) { + if(false == np_rd_receive(s)) { H5_FAILED(); AT(); - goto error; + goto error2; } decisleep(s->tick_len * s->update_interval); dbgf(1, "reader: finish reading the message: %d\n",s->np_notify); } + + if (which >= s->nsteps) { + H5_FAILED(); AT(); + printf("Number of created groups is out of bounds\n"); + goto error; + } + + esnprintf(name, sizeof(name), "/group-%d", which); + if((g = H5Gopen(s->file, name, H5P_DEFAULT)) <0) { H5_FAILED(); AT(); printf("H5Gopen failed\n"); - if(s->use_named_pipes && s->attr_test == true) { - dbgf(1, "Reader: the H5Gopen verfication failed for group %s \n",name); - np_send_error(s,false); - } goto error; } + /* Reader sends an OK message back to the reader */ if(s->use_named_pipes && s->attr_test == true) { + if(np_rd_send(s)==false) goto error; dbgf(1, "Reader: finish sending back the message: %d\n",s->np_notify); } + + /* Check if we need to skip the attribute test for this group. */ if (s->asteps != 0 && which % s->asteps == 0) result = verify_group_attribute(s, g, which); else @@ -2383,11 +2734,19 @@ verify_group(state_t *s, unsigned int which) } return result; + error: + H5E_BEGIN_TRY { H5Gclose(g); } H5E_END_TRY; + /* The reader sends an error message to the writer to stop the test.*/ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,false); + +error2: + return false; } @@ -2494,6 +2853,7 @@ main(int argc, char **argv) goto error; } + /* Pass the named pipe information to the struct of state_t s, for attribute tests.*/ if(s.use_named_pipes) { s.np_fd_w_to_r = fd_writer_to_reader; s.np_fd_r_to_w = fd_reader_to_writer; @@ -2503,12 +2863,13 @@ main(int argc, char **argv) s.max_lag = config.max_lag; } - /* For attribute test, force the named pipe to communicate for every step. */ + /* For attribute test, force the named pipe to communicate in every step. */ if (s.at_pattern != ' ') { s.attr_test = true; if(s.use_named_pipes) s.csteps = 1; } + if (writer) { for (step = 0; step < s.nsteps; step++) { dbgf(2, "writer: step %d\n", step); @@ -2549,7 +2910,7 @@ main(int argc, char **argv) } } - /* For the default test, wait for a few ticks for the update to happen */ + /* For the default test, wait for a few ticks for the update to happen */ if(s.use_named_pipes && s.attr_test== false) decisleep(config.tick_len * s.update_interval); From 25c3a55b652ddb9a9d901c3913d164e249639b6a Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Fri, 23 Apr 2021 14:10:01 -0500 Subject: [PATCH 07/13] Use -q option for the group attribute test. --- test/testvfdswmr.sh.in | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index b810a8dbe46..9e75aebd482 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -155,7 +155,6 @@ if [ $rc -ne 0 ] ; then exit 0 fi -<<<<<<< HEAD all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo groups attrdset" all_tests="${all_tests} groups_attrs few_big many_small" tests=${all_tests} @@ -757,13 +756,11 @@ for options in ${grp_attr_list[*]}; do fi echo launch vfd_swmr_group attribute: $options catch_out_err_and_rc vfd_swmr_group_writer \ - ../vfd_swmr_group_writer -c 1 -n $GROUP_attr_n -a 1 -A $options & - #../vfd_swmr_group_writer -q -c 1 -n $GROUP_attr_n -a 1 -A $options & + ../vfd_swmr_group_writer -q -c 1 -n $GROUP_attr_n -a 1 -A $options & pid_writer=$! catch_out_err_and_rc vfd_swmr_group_reader \ - ../vfd_swmr_group_reader -c 1 -n $GROUP_attr_n -a 1 -A $options & - #../vfd_swmr_group_reader -q -c 1 -n $GROUP_attr_n -a 1 -A $options & + ../vfd_swmr_group_reader -q -c 1 -n $GROUP_attr_n -a 1 -A $options & pid_reader=$! # Wait for the reader to finish before signalling the From 7aa93ccf61135780210e539dde4e1eb6c7d22c3e Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Mon, 26 Apr 2021 10:12:08 -0500 Subject: [PATCH 08/13] Forgot passing group property list when merging the code --- test/vfd_swmr_group_writer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 7ffb0488cfe..fc21e686704 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -1605,7 +1605,7 @@ write_group(state_t *s, unsigned int which) } } - if ((g = H5Gcreate2(s->file, name, H5P_DEFAULT, H5P_DEFAULT, + if ((g = H5Gcreate2(s->file, name, H5P_DEFAULT, gcpl, H5P_DEFAULT)) < 0) { H5_FAILED(); AT(); printf("H5Gcreate2 failed\n"); From e70444ccd2ccc474f221114e6dc74dfa0695063d Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Mon, 26 Apr 2021 16:24:23 -0500 Subject: [PATCH 09/13] Add the old-styled group creation attribute tests. --- test/testvfdswmr.sh.in | 67 ++++++++++++++- test/vfd_swmr_common.c | 4 + test/vfd_swmr_group_writer.c | 153 ++++++++++++++++++++++++++--------- 3 files changed, 184 insertions(+), 40 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index 9e75aebd482..06b1023f007 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -156,7 +156,7 @@ if [ $rc -ne 0 ] ; then fi all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo groups attrdset" -all_tests="${all_tests} groups_attrs few_big many_small" +all_tests="${all_tests} groups_attrs os_groups_attrs few_big many_small" tests=${all_tests} if [ $# -gt 0 ]; then @@ -787,6 +787,71 @@ for options in ${grp_attr_list[*]}; do rm -f vfd_swmr_group_reader.*.{out,rc} done +# The following tests are for add/del/modify attributes for +# groups created with the old-style. +# Check https://portal.hdfgroup.org/display/HDF5/Groups for +# the detailed group implementation note. +# The 'compact' and 'compact-del' are the attribute addition +# and deletion tests. Other test names have the same meaning +# as those of the new-style group tests. +# +os_grp_attr_list=( + "compact" + "compact-del" + "modify" + "vstr" + "remove-vstr" + "modify-vstr" + ) +os_grp_sub_attr_list=( + "modify" + "remove-vstr" + "modify-vstr" + ) +if [[ "$HDF5TestExpress" -gt 0 ]] ; then #Setting for standard run + os_grp_attr_list=("${os_grp_sub_attr_list[@]}") +fi + +for options in ${os_grp_attr_list[*]}; do + if [ ${do_os_groups_attrs:-no} = no ]; then + continue + fi + echo launch vfd_swmr_group attribute with old-style group: $options + catch_out_err_and_rc vfd_swmr_group_writer \ + ../vfd_swmr_group_writer -q -G -c 1 -n $GROUP_attr_n -a 1 -A $options & + pid_writer=$! + + catch_out_err_and_rc vfd_swmr_group_reader \ + ../vfd_swmr_group_reader -q -G -c 1 -n $GROUP_attr_n -a 1 -A $options & + pid_reader=$! + + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. + wait $pid_reader + wait $pid_writer + + # Collect exit code of the reader + if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + + # Collect exit code of the writer + if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_group_writer.{out,rc} + rm -f vfd_swmr_group_reader.*.{out,rc} +done + + + + for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do if [ ${do_many_small:-no} = no ]; then continue diff --git a/test/vfd_swmr_common.c b/test/vfd_swmr_common.c index 68e0b9c47a4..aed9cbe5e02 100644 --- a/test/vfd_swmr_common.c +++ b/test/vfd_swmr_common.c @@ -391,6 +391,10 @@ vfd_swmr_create_fapl(bool use_latest_format, bool use_vfd_swmr, bool only_meta_p if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) return H5I_INVALID_HID; } + else {/* Currently this is used only for old-styled group implementation tests.*/ + if (H5Pset_libver_bounds(fapl, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0) + return H5I_INVALID_HID; + } /* Enable page buffering */ if (H5Pset_page_buffer_size(fapl, 4096, only_meta_pages ? 100 : 0, 0) < 0) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index fc21e686704..2477c2ebfc0 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -35,6 +35,7 @@ typedef struct { unsigned int nsteps; unsigned int update_interval; bool use_vfd_swmr; + bool old_style_grp; bool use_named_pipes; char at_pattern; bool attr_test; @@ -52,6 +53,7 @@ typedef struct { .file = H5I_INVALID_HID, .one_by_one_sid = H5I_INVALID_HID, .filename = "", \ .filetype = H5T_NATIVE_UINT32, .asteps = 10, .csteps = 10, .nsteps = 100, .update_interval = READER_WAIT_TICKS, \ .use_vfd_swmr = true, \ + .old_style_grp = false, \ .use_named_pipes = true \ , .at_pattern = ' ' \ , .attr_test = false \ @@ -67,10 +69,11 @@ typedef struct { static void usage(const char *progname) { - fprintf(stderr, "usage: %s [-S] [-a steps] [-b] [-c]\n" + fprintf(stderr, "usage: %s [-S] [-G] [-a steps] [-b] [-c]\n" " [-n iterations] [-N] [-q] [-u numb_ticks] [-A at_pattern]\n" "\n" "-S: do not use VFD SWMR\n" + "-G: old-style type of group\n" "-a steps: `steps` between adding attributes\n" "-b: write data in big-endian byte order\n" "-c steps: `steps` between communication between the writer and reader\n" @@ -127,11 +130,14 @@ state_init(state_t *s, int argc, char **argv) if (tfile) HDfree(tfile); - while ((ch = getopt(argc, argv, "Sa:bc:n:Nqu:A:")) != -1) { + while ((ch = getopt(argc, argv, "SGa:bc:n:Nqu:A:")) != -1) { switch (ch) { case 'S': s->use_vfd_swmr = false; break; + case 'G': + s->old_style_grp = true; + break; case 'a': case 'c': case 'n': @@ -1080,13 +1086,17 @@ add_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { unsigned min_dense = 0; const char* aname_format="attr-%u-%u"; - /* Obtain the maximal number of attributes to be stored in compact - * storage and the minimal number of attributes to be stored in - * dense storage. */ - if(H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense)<0) { - H5_FAILED(); AT(); - printf("H5Pget_attr_phase_change() failed\n"); - goto error; + if(s->old_style_grp) + max_compact = 2; + else { + /* Obtain the maximal number of attributes to be stored in compact + * storage and the minimal number of attributes to be stored in + * dense storage. */ + if(H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense)<0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change() failed\n"); + goto error; + } } /* Add max_compact attributes, these attributes are stored in @@ -1580,6 +1590,7 @@ write_group(state_t *s, unsigned int which) hid_t g = H5I_INVALID_HID; hid_t gcpl = H5I_INVALID_HID; bool result = true; + H5G_info_t group_info; if (which >= s->nsteps) { H5_FAILED(); AT(); @@ -1589,20 +1600,24 @@ write_group(state_t *s, unsigned int which) esnprintf(name, sizeof(name), "/group-%d", which); - gcpl = H5Pcreate(H5P_GROUP_CREATE); - if(gcpl <0) { - H5_FAILED(); AT(); - printf("H5Pcreate failed\n"); - goto error; - } - - /* If we test the dense storage, change the attribute phase. */ - if(s->at_pattern =='d') { - if(H5Pset_attr_phase_change(gcpl, 0, 0) <0) { + if(s->old_style_grp) + gcpl = H5P_DEFAULT; + else { + gcpl = H5Pcreate(H5P_GROUP_CREATE); + if(gcpl <0) { H5_FAILED(); AT(); - printf("H5Pset_attr_phase_change failed for the dense storage.\n"); + printf("H5Pcreate failed\n"); goto error; } + + /* If we test the dense storage, change the attribute phase. */ + if(s->at_pattern =='d') { + if(H5Pset_attr_phase_change(gcpl, 0, 0) <0) { + H5_FAILED(); AT(); + printf("H5Pset_attr_phase_change failed for the dense storage.\n"); + goto error; + } + } } if ((g = H5Gcreate2(s->file, name, H5P_DEFAULT, gcpl, @@ -1612,6 +1627,29 @@ write_group(state_t *s, unsigned int which) goto error; } + if(H5Gget_info(g,&group_info) <0) { + H5_FAILED(); AT(); + printf("H5Gget_info failed\n"); + goto error; + } + + if(s->old_style_grp) { + if(group_info.storage_type != H5G_STORAGE_TYPE_SYMBOL_TABLE) { + H5_FAILED(); AT(); + printf("Old-styled group test: but the group is not in old-style. \n"); + goto error; + } + dbgf(2,"Writer: group is created with the old-style.\n"); + } + else { + if(group_info.storage_type == H5G_STORAGE_TYPE_SYMBOL_TABLE) { + H5_FAILED(); AT(); + printf("The created group should NOT be in old-style . \n"); + goto error; + } + dbgf(2,"Writer: group is created with the new-style.\n"); + + } /* If an attribute test is turned on and named pipes are used, * the writer should send and receive messages after the group creation. * This will distinguish an attribute operation error from an @@ -1639,7 +1677,7 @@ write_group(state_t *s, unsigned int which) goto error; } - if(H5Pclose(gcpl) <0) { + if(!s->old_style_grp && H5Pclose(gcpl) <0) { H5_FAILED(); AT(); printf("H5Pclose failed\n"); goto error; @@ -1656,7 +1694,8 @@ write_group(state_t *s, unsigned int which) H5E_BEGIN_TRY { H5Gclose(g); - H5Pclose(gcpl); + if(!s->old_style_grp) + H5Pclose(gcpl); } H5E_END_TRY; return false; @@ -2571,27 +2610,33 @@ verify_group_attribute(state_t *s, hid_t g, unsigned int which) * "dense-del", "dense-del-to-compact", * the maximal number of attributes for the compact storage * and the minimal number of attributes for the dense storage - * are needed. So obtain them here */ + * are needed. So obtain them here + * When testing the old-style group creation case, only max_compact + * matters. To reduce the testing time, we set max_compact to 2.*/ switch (test_pattern) { case 'c': case 't': case 'C': case 'D': case 'T': - if((gcpl = H5Gget_create_plist(g)) < 0) { - H5_FAILED(); AT(); - printf("H5Gget_create_plist failed\n"); - goto error; - } - if (H5Pget_attr_phase_change(gcpl,&max_compact,&min_dense) < 0) { - H5_FAILED(); AT(); - printf("H5Pget_attr_phase_change failed\n"); - goto error; - } - if(H5Pclose(gcpl) < 0) { - H5_FAILED(); AT(); - printf("H5Pclose failed\n"); - goto error; + if(s->old_style_grp) + max_compact = 2; + else { + if((gcpl = H5Gget_create_plist(g)) < 0) { + H5_FAILED(); AT(); + printf("H5Gget_create_plist failed\n"); + goto error; + } + if (H5Pget_attr_phase_change(gcpl,&max_compact,&min_dense) < 0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change failed\n"); + goto error; + } + if(H5Pclose(gcpl) < 0) { + H5_FAILED(); AT(); + printf("H5Pclose failed\n"); + goto error; + } } break; case 'v': @@ -2681,6 +2726,7 @@ verify_group(state_t *s, unsigned int which) char name[sizeof("/group-9999999999")]; hid_t g = H5I_INVALID_HID; bool result = true; + H5G_info_t group_info; /* The reader receives a message from the writer.Then sleep * for a few ticks or stop the test if the received message @@ -2712,6 +2758,31 @@ verify_group(state_t *s, unsigned int which) goto error; } + if(H5Gget_info(g,&group_info) <0) { + H5_FAILED(); AT(); + printf("H5Gget_info failed\n"); + goto error; + } + + dbgf(2,"Storage info is %d\n",group_info.storage_type); + if(s->old_style_grp) { + if(group_info.storage_type != H5G_STORAGE_TYPE_SYMBOL_TABLE) { + H5_FAILED(); AT(); + printf("Reader - Old-styled group: but the group is not in old-style. \n"); + goto error; + } + dbgf(2,"Reader: verify that the group is created with the old-style.\n"); + } + else { + if(group_info.storage_type == H5G_STORAGE_TYPE_SYMBOL_TABLE) { + H5_FAILED(); AT(); + printf("Reader - The created group should NOT be in old-style . \n"); + goto error; + } + dbgf(2,"Reader: verify that the group is created with the new-style.\n"); + + } + /* Reader sends an OK message back to the reader */ if(s->use_named_pipes && s->attr_test == true) { @@ -2790,8 +2861,12 @@ main(int argc, char **argv) /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ init_vfd_swmr_config(&config, 4, 7, writer, FALSE, 128, "./group-shadow"); - /* use_latest_format, use_vfd_swmr, only_meta_page, config */ - if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config)) < 0) { + /* If old-style option is chosen, use the earliest file format(H5F_LIBVER_EARLIEST) + * as the second parameter of H5Pset_libver_bound() that is called by + * vfd_swmr_create_fapl. Otherwise, the latest file format(H5F_LIBVER_LATEST) + * should be used as the second parameter of H5Pset_libver_bound(). + * Also pass the use_vfd_swmr, only_meta_page, config to vfd_swmr_create_fapl().*/ + if ((fapl = vfd_swmr_create_fapl(!s.old_style_grp, s.use_vfd_swmr, true, &config)) < 0) { H5_FAILED(); AT(); printf("vfd_swmr_create_fapl failed\n"); goto error; From 87482f46c21e16bccc7484ffa36701ca237b996c Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Thu, 29 Apr 2021 11:31:34 -0500 Subject: [PATCH 10/13] 1. Add the attribute storage check 2. Change the test names and remove unnecessary debugging information Tested at jelly. --- test/testvfdswmr.sh.in | 6 +- test/vfd_swmr_group_writer.c | 138 +++++++++++++++++++++++++++++------ 2 files changed, 120 insertions(+), 24 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index 06b1023f007..979f5a96861 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -721,12 +721,12 @@ fi grp_attr_list=( "compact" "dense" - "compact-to-dense" "compact-del" "dense-del" + "compact-add-to-dense" "dense-del-to-compact" "modify" - "vstr" + "add-vstr" "remove-vstr" "modify-vstr" ) @@ -799,7 +799,7 @@ os_grp_attr_list=( "compact" "compact-del" "modify" - "vstr" + "add-vstr" "remove-vstr" "modify-vstr" ) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 2477c2ebfc0..85085da736e 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -65,7 +65,6 @@ typedef struct { , .np_verify = 0 } -//TODO: add at_pattern description static void usage(const char *progname) { @@ -85,19 +84,19 @@ usage(const char *progname) " The value of `at_pattern` is one of the following:\n" " `compact` - Attributes added in compact storage\n" " `dense` - An attribute added in dense storage\n" - " `compact-to-dense` - Attributes added first in compact\n" - " then in dense storage\n" " `compact-del` - Attributes added and then one\n" " attribute deleted, in compact \n" " `dense-del` - Attributes added until the storage\n" " is dense then an attribute deleted\n" " the storge still in dense\n" + " `compact-add-to-dense` - Attributes added first in compact\n" + " then in dense storage\n" " `dense-del-to-compact` - Attributes added until the storage\n" " is dense, then several attributes \n" " deleted, the storage changed to\n" " compact\n" " `modify` - An attribute added then modified\n" - " `vstr` - A VL string attribute added\n" + " `add-vstr` - A VL string attribute added\n" " `remove-vstr` - A VL string attribute added then\n" " deleted\n" " `modify-vstr` - A VL string attribute added then \n" @@ -178,7 +177,7 @@ state_init(state_t *s, int argc, char **argv) s->at_pattern = 'c'; else if (strcmp(optarg, "dense") == 0) s->at_pattern = 'd'; - else if (strcmp(optarg, "compact-to-dense") == 0) + else if (strcmp(optarg, "compact-add-to-dense") == 0) s->at_pattern = 't'; else if (strcmp(optarg, "compact-del") == 0) s->at_pattern = 'C'; @@ -188,7 +187,7 @@ state_init(state_t *s, int argc, char **argv) s->at_pattern = 'T'; else if (strcmp(optarg, "modify") == 0) s->at_pattern = 'M'; - else if (strcmp(optarg,"vstr") ==0) + else if (strcmp(optarg,"add-vstr") ==0) s->at_pattern = 'v'; else if (strcmp(optarg, "remove-vstr") == 0) s->at_pattern = 'r'; @@ -1225,10 +1224,6 @@ del_attrs_compact_dense_compact(state_t *s, } u= max_compact +1; -#if 0 - if(max_compact < min_dense) - printf("Minimum number of attributes stored in dense storage should be less than maximum number of attributes stored in compact storage.\n"); -#endif // delete a number of attributes so that the attribute storage just becomes dense. for(u--;u>=(min_dense-1);u--) { @@ -1701,6 +1696,62 @@ write_group(state_t *s, unsigned int which) return false; } +/*------------------------------------------------------------------------- + * Function: check_attr_storage_type + * + * Purpose: Check if the attribute storage type is correct + * + * Parameters: hid_t oid + * HDF5 object ID (in this file: means group ID) + * + * bool is_compact + * true if the attribute is stored in compact storage + * false if the attribute is stored in dense storage + * + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ + +static bool +check_attr_storage_type(hid_t g, + bool is_compact) { + + H5O_native_info_t ninfo; + + /* Get the object information */ + if(H5Oget_native_info(g, &ninfo, H5O_NATIVE_INFO_HDR|H5O_NATIVE_INFO_META_SIZE) < 0) { + H5_FAILED(); AT(); + printf("H5Oget_native_info failed\n"); + goto error; + } + + if(is_compact) { + if(ninfo.meta_size.attr.index_size != 0 || + ninfo.meta_size.attr.heap_size != 0) { + H5_FAILED(); AT(); + printf("Should be in compact storage,but it is not.\n"); + goto error; + } + } + else { + if(ninfo.meta_size.attr.index_size == 0 || + ninfo.meta_size.attr.heap_size == 0) { + H5_FAILED(); AT(); + printf("Should be in dense storage,but it is not.\n"); + goto error; + } + } + + return true; + +error: + return false; + +} + /*------------------------------------------------------------------------- * Function: vrfy_attr @@ -1725,6 +1776,16 @@ write_group(state_t *s, unsigned int which) * This parameter is used to generate correct group name in a key * debugging message. * + * bool check_storage + * a flag to indicate if the storage check is on + * + * bool is_compact + * true if the attribute is stored in compact storage + * false if the attribute is stored in dense storage + * Note: this parameter is not used if the check_storage + * is set to false. + * + * * Return: Success: true * Failure: false * @@ -1736,7 +1797,9 @@ vrfy_attr(state_t *s, hid_t g, unsigned int which, const char* aname, - unsigned int g_which) { + unsigned int g_which, + bool check_storage, + bool is_compact) { unsigned int read_which; hid_t aid = H5I_INVALID_HID; @@ -1793,6 +1856,17 @@ vrfy_attr(state_t *s, goto error; } + + if(!s->old_style_grp && check_storage == true) { + if(false == check_attr_storage_type(g,is_compact)) { + H5_FAILED(); AT(); + printf("The attribute storage type is wrong. \n"); + goto error; + } + dbgf(2, "reader: finish checking the storage type: %d\n", s->np_notify); + + } + /* If the read value is expected, send back an OK message to the writer. */ if(s->use_named_pipes && s->attr_test == true) { if(np_rd_send(s)==false) @@ -1846,7 +1920,7 @@ verify_default_group_attr(state_t*s,hid_t g, unsigned int which) char attrname[VS_ATTR_NAME_LEN]; const char* aname_format = "attr-%u"; HDsprintf(attrname, aname_format, which); - return vrfy_attr(s,g,which,attrname,which); + return vrfy_attr(s,g,which,attrname,which,false,true); } @@ -2150,7 +2224,12 @@ verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) static bool -verify_del_one_attr(state_t *s,hid_t g, const char *aname) { +verify_del_one_attr(state_t *s, + hid_t g, + const char *aname, + bool check_storage, + bool is_compact) { + htri_t attr_exists = FALSE; @@ -2182,6 +2261,16 @@ verify_del_one_attr(state_t *s,hid_t g, const char *aname) { goto error; } + if(!s->old_style_grp && check_storage == true) { + if(false == check_attr_storage_type(g,is_compact)) { + H5_FAILED(); AT(); + printf("The attribute storage type is wrong. \n"); + goto error; + } + dbgf(2, "reader: finish checking the storage type: %d\n", s->np_notify); + + } + /* Reader sends an OK message back to the reader */ if(s->use_named_pipes && s->attr_test == true) { if(np_rd_send(s)==false) @@ -2238,7 +2327,7 @@ verify_remove_vlstr_attr(state_t* s,hid_t g, unsigned int which) ret = verify_group_vlstr_attr(s,g,which,false); if(ret == true) { HDsprintf(attrname,aname_format,which); - ret = verify_del_one_attr(s,g,attrname); + ret = verify_del_one_attr(s,g,attrname,false,false); } return ret; } @@ -2326,7 +2415,7 @@ verify_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { for (u = 0; u < max_c; u++) { HDsprintf(attrname, aname_format, which,u); - if(false == vrfy_attr(s,g,u+which,attrname,which)) { + if(false == vrfy_attr(s,g,u+which,attrname,which,true,true)) { ret = false; break; } @@ -2380,7 +2469,7 @@ verify_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int whi /* Now the storage is in dense. Verify if the * retrieved value is correct. */ HDsprintf(attrname, aname_format, max_c+which,0); - ret = vrfy_attr(s,g,which+max_c,attrname,which); + ret = vrfy_attr(s,g,which+max_c,attrname,which,true,false); if(ret == false) dbgf(1,"verify_attrs_compact_dense failed \n"); @@ -2432,7 +2521,7 @@ verify_del_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which if(ret == true) { /* The writer only deletes the attribute attr-which-0 */ HDsprintf(attrname,aname_format,which,0); - ret = verify_del_one_attr(s,g,attrname); + ret = verify_del_one_attr(s,g,attrname,true,true); } return ret; @@ -2487,7 +2576,7 @@ verify_del_attrs_compact_dense(state_t *s, if(ret == true) { /* The writer only deletes the attribute attr-d-which-0 */ HDsprintf(attrname,aname_format,max_c+which,0); - ret = verify_del_one_attr(s,g,attrname); + ret = verify_del_one_attr(s,g,attrname,true,false); } return ret; @@ -2558,13 +2647,16 @@ verify_del_attrs_compact_dense_compact(state_t *s, u = max_c + 1; for(u--;u>=(min_d-1);u--) { HDsprintf(attrname, aname_format, which,max_c-u); - ret = verify_del_one_attr(s,g,attrname); + if(u==(min_d-1)) + ret = verify_del_one_attr(s,g,attrname,true,true); + else + ret = verify_del_one_attr(s,g,attrname,true,false); } /* Just verify one more deleted attribute by the writer. The storage is still compact. */ HDsprintf(attrname,aname_format,max_c+which,0); - ret = verify_del_one_attr(s,g,attrname); + ret = verify_del_one_attr(s,g,attrname,true,true); } return ret; @@ -2938,7 +3030,11 @@ main(int argc, char **argv) s.max_lag = config.max_lag; } - /* For attribute test, force the named pipe to communicate in every step. */ + /* For attribute test, force the named pipe to communicate in every step. + * This will avoid the fake verification error from the reader when using the named pipe. + * If the named pipe is not forced to communicate in every step, the reader may go ahead + * to verify the group and the attribute operations before the writer has a chance to + * carry out the corresponding operations. */ if (s.at_pattern != ' ') { s.attr_test = true; if(s.use_named_pipes) From 226002d5835db5bad2cdf853c50fe918b3db1bdb Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Thu, 29 Apr 2021 15:58:23 -0500 Subject: [PATCH 11/13] Use /* */ instead of // for comments. --- test/vfd_swmr_group_writer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 85085da736e..fb175ccc096 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -1225,7 +1225,7 @@ del_attrs_compact_dense_compact(state_t *s, u= max_compact +1; - // delete a number of attributes so that the attribute storage just becomes dense. + /* delete a number of attributes so that the attribute storage just becomes dense.*/ for(u--;u>=(min_dense-1);u--) { HDsprintf(attrname, aname_format, which,max_compact-u); if (H5Adelete(obj_id,attrname) < 0) { From 5bac3e28f0c21c57c26990ea707a6383fa637e37 Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Thu, 29 Apr 2021 18:31:13 -0500 Subject: [PATCH 12/13] Replace all string functions with HDF's corresponding string functions. --- test/vfd_swmr_group_writer.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index fb175ccc096..cb6b0863fc4 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -142,7 +142,7 @@ state_init(state_t *s, int argc, char **argv) case 'n': case 'u': errno = 0; - tmp = strtoul(optarg, &end, 0); + tmp = HDstrtoul(optarg, &end, 0); if (end == optarg || *end != '\0') { H5_FAILED(); AT(); printf("couldn't parse `-%c` argument `%s`\n", ch, optarg); @@ -173,25 +173,25 @@ state_init(state_t *s, int argc, char **argv) s->use_named_pipes = false; break; case 'A': - if (strcmp(optarg, "compact") == 0) + if (HDstrcmp(optarg, "compact") == 0) s->at_pattern = 'c'; - else if (strcmp(optarg, "dense") == 0) + else if (HDstrcmp(optarg, "dense") == 0) s->at_pattern = 'd'; - else if (strcmp(optarg, "compact-add-to-dense") == 0) + else if (HDstrcmp(optarg, "compact-add-to-dense") == 0) s->at_pattern = 't'; - else if (strcmp(optarg, "compact-del") == 0) + else if (HDstrcmp(optarg, "compact-del") == 0) s->at_pattern = 'C'; - else if (strcmp(optarg, "dense-del") == 0) + else if (HDstrcmp(optarg, "dense-del") == 0) s->at_pattern = 'D'; - else if (strcmp(optarg, "dense-del-to-compact") == 0) + else if (HDstrcmp(optarg, "dense-del-to-compact") == 0) s->at_pattern = 'T'; - else if (strcmp(optarg, "modify") == 0) + else if (HDstrcmp(optarg, "modify") == 0) s->at_pattern = 'M'; - else if (strcmp(optarg,"add-vstr") ==0) + else if (HDstrcmp(optarg,"add-vstr") ==0) s->at_pattern = 'v'; - else if (strcmp(optarg, "remove-vstr") == 0) + else if (HDstrcmp(optarg, "remove-vstr") == 0) s->at_pattern = 'r'; - else if (strcmp(optarg, "modify-vstr") == 0) + else if (HDstrcmp(optarg, "modify-vstr") == 0) s->at_pattern = 'm'; else { H5_FAILED(); AT(); @@ -2936,13 +2936,13 @@ main(int argc, char **argv) goto error; } - personality = strstr(s.progname, "vfd_swmr_group_"); + personality = HDstrstr(s.progname, "vfd_swmr_group_"); if (personality != NULL && - strcmp(personality, "vfd_swmr_group_writer") == 0) + HDstrcmp(personality, "vfd_swmr_group_writer") == 0) writer = true; else if (personality != NULL && - strcmp(personality, "vfd_swmr_group_reader") == 0) + HDstrcmp(personality, "vfd_swmr_group_reader") == 0) writer = false; else { H5_FAILED(); AT(); From 16b2b76b263afe59bb001df80b04222471ee9cc1 Mon Sep 17 00:00:00 2001 From: Muqun Yang Date: Fri, 30 Apr 2021 09:10:01 -0500 Subject: [PATCH 13/13] Close the space id used for attributes. --- test/vfd_swmr_group_writer.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index cb6b0863fc4..7f083c8a52a 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -3123,6 +3123,12 @@ main(int argc, char **argv) goto error; } + if (H5Sclose(s.one_by_one_sid) < 0) { + H5_FAILED(); AT(); + printf("H5Sclose failed\n"); + goto error; + } + if (H5Fclose(s.file) < 0) { H5_FAILED(); AT(); printf("H5Fclose failed\n"); @@ -3163,6 +3169,7 @@ main(int argc, char **argv) H5E_BEGIN_TRY { H5Pclose(fapl); H5Pclose(fcpl); + H5Sclose(s.one_by_one_sid); H5Fclose(s.file); } H5E_END_TRY;