From e7796a60d2e53a84417f865fd2918c386fe719ae Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Wed, 19 Jun 2024 07:38:30 -0500 Subject: [PATCH] Add page buffer cache command line option to tools (#4562) Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- release_docs/RELEASE.txt | 8 ++ tools/lib/h5diff.c | 70 +++++++---- tools/lib/h5diff.h | 1 + tools/lib/h5tools.c | 45 ++++--- tools/lib/h5tools.h | 5 +- tools/libtest/h5tools_test_utils.c | 11 +- tools/src/h5diff/h5diff_common.c | 12 +- tools/src/h5dump/h5dump.c | 40 ++++++- tools/src/h5ls/h5ls.c | 52 +++++--- tools/src/h5repack/h5repack.c | 6 +- tools/src/h5repack/h5repack.h | 6 +- tools/src/h5repack/h5repack_copy.c | 6 +- tools/src/h5repack/h5repack_main.c | 112 +++++++++++------- tools/src/h5repack/h5repack_verify.c | 10 +- tools/src/h5stat/h5stat.c | 30 ++++- tools/src/misc/h5mkgrp.c | 70 ++++++----- tools/test/h5diff/expected/h5diff_10.txt | 1 + tools/test/h5diff/expected/h5diff_600.txt | 1 + tools/test/h5diff/expected/h5diff_603.txt | 1 + tools/test/h5diff/expected/h5diff_606.txt | 1 + tools/test/h5diff/expected/h5diff_612.txt | 1 + tools/test/h5diff/expected/h5diff_615.txt | 1 + tools/test/h5diff/expected/h5diff_621.txt | 1 + tools/test/h5diff/expected/h5diff_622.txt | 1 + tools/test/h5diff/expected/h5diff_623.txt | 1 + tools/test/h5diff/expected/h5diff_624.txt | 1 + tools/test/h5dump/CMakeTests.cmake | 2 + .../test/h5dump/expected/file_space_cache.ddl | 22 ++++ tools/test/h5dump/expected/h5dump-help.txt | 1 + .../pbits/tnofilename-with-packed-bits.ddl | 1 + .../expected/pbits/tpbitsIncomplete.ddl | 1 + .../expected/pbits/tpbitsLengthExceeded.ddl | 1 + .../expected/pbits/tpbitsLengthPositive.ddl | 1 + .../expected/pbits/tpbitsMaxExceeded.ddl | 1 + .../expected/pbits/tpbitsOffsetExceeded.ddl | 1 + .../expected/pbits/tpbitsOffsetNegative.ddl | 1 + tools/test/h5dump/testh5dump.sh.in | 2 + tools/test/h5ls/CMakeTests.cmake | 3 + tools/test/h5ls/expected/help-1.ls | 1 + tools/test/h5ls/expected/help-2.ls | 1 + tools/test/h5ls/expected/help-3.ls | 1 + .../h5ls/expected/textlinksrc-nodangle-1.ls | 1 + tools/test/h5ls/expected/tgroup-1.ls | 1 + .../test/h5repack/expected/h5repack-help.txt | 1 + tools/test/h5repack/h5repacktst.c | 10 +- tools/test/h5stat/CMakeTests.cmake | 3 + .../h5stat/expected/h5stat_filters-cache.ddl | 99 ++++++++++++++++ tools/test/h5stat/expected/h5stat_help1.ddl | 1 + tools/test/h5stat/expected/h5stat_help2.ddl | 1 + tools/test/h5stat/expected/h5stat_nofile.ddl | 1 + tools/test/h5stat/testh5stat.sh.in | 3 + 51 files changed, 495 insertions(+), 160 deletions(-) create mode 100644 tools/test/h5dump/expected/file_space_cache.ddl create mode 100644 tools/test/h5stat/expected/h5stat_filters-cache.ddl diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index afbc3b5b83c..8b43f5ecddb 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -765,6 +765,14 @@ New Features Tools: ------ + - Add option to adjust the page buffer size in tools + + The page buffer cache size for a file can now be adjusted using the + --page-buffer-size=N + option in the h5repack, h5diff, h5dump, h5ls, and h5stat tools. This + will call the H5Pset_page_buffer_size() API function with the specified + size in bytes. + - Allow h5repack to reserve space for a user block without a file This is useful for users who want to reserve space diff --git a/tools/lib/h5diff.c b/tools/lib/h5diff.c index 9329872d965..bd8112f2c65 100644 --- a/tools/lib/h5diff.c +++ b/tools/lib/h5diff.c @@ -624,46 +624,72 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char *------------------------------------------------------------------------- */ /* open file 1 */ - if (opts->vfd_info[0].u.name) { - if ((fapl1_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[0]))) < 0) { - parallel_print("h5diff: unable to create fapl for input file\n"); - H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n"); + if ((fapl1_id = h5tools_get_new_fapl(H5P_DEFAULT)) < 0) { + parallel_print("h5diff: unable to create fapl for input file\n"); + H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n"); + } + + /* Set non-default virtual file driver, if requested */ + if (opts->custom_vfd[0] && opts->vfd_info[0].u.name) { + if (h5tools_set_fapl_vfd(fapl1_id, &(opts->vfd_info[0])) < 0) { + parallel_print("h5diff: unable to set VFD on fapl for input file\n"); + H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "failed to set VFD on FAPL\n"); + } + } + + /* Set non-default VOL connector, if requested */ + if (opts->custom_vol[0]) { + if (h5tools_set_fapl_vol(fapl1_id, &(opts->vol_info[0])) < 0) { + parallel_print("h5diff: unable to set VOL on fapl for input file\n"); + H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "failed to set VOL on FAPL\n"); } } - if (opts->custom_vol[0] || opts->custom_vfd[0]) { - if ((fapl1_id = h5tools_get_fapl(fapl1_id, opts->custom_vol[0] ? &(opts->vol_info[0]) : NULL, - opts->custom_vfd[0] ? &(opts->vfd_info[0]) : NULL)) < 0) { - parallel_print("h5diff: unable to create fapl for input file\n"); - H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n"); + if (opts->page_cache > 0) { + if (H5Pset_page_buffer_size(fapl1_id, opts->page_cache, 0, 0) < 0) { + parallel_print("h5diff: unable to set page buffer cache size for fapl for input file\n"); + H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to set page buffer cache size on FAPL\n"); } } - if ((file1_id = h5tools_fopen(fname1, H5F_ACC_RDONLY, fapl1_id, (fapl1_id != H5P_DEFAULT), NULL, - (size_t)0)) < 0) { + if ((file1_id = h5tools_fopen(fname1, H5F_ACC_RDONLY, fapl1_id, + (opts->custom_vol[0] || opts->custom_vfd[0]), NULL, (size_t)0)) < 0) { parallel_print("h5diff: <%s>: unable to open file\n", fname1); H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "<%s>: unable to open file\n", fname1); } H5TOOLS_DEBUG("file1_id = %s", fname1); /* open file 2 */ - if (opts->vfd_info[1].u.name) { - if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[1]))) < 0) { - parallel_print("h5diff: unable to create fapl for output file\n"); - H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n"); + if ((fapl2_id = h5tools_get_new_fapl(H5P_DEFAULT)) < 0) { + parallel_print("h5diff: unable to create fapl for output file\n"); + H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n"); + } + + /* Set non-default virtual file driver, if requested */ + if (opts->custom_vfd[1] && opts->vfd_info[1].u.name) { + if (h5tools_set_fapl_vfd(fapl2_id, &(opts->vfd_info[1])) < 0) { + parallel_print("h5diff: unable to set VFD on fapl for output file\n"); + H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "failed to set VFD on FAPL\n"); + } + } + + /* Set non-default VOL connector, if requested */ + if (opts->custom_vol[1]) { + if (h5tools_set_fapl_vol(fapl2_id, &(opts->vol_info[1])) < 0) { + parallel_print("h5diff: unable to set VOL on fapl for output file\n"); + H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "failed to set VOL on FAPL\n"); } } - if (opts->custom_vol[1] || opts->custom_vfd[1]) { - if ((fapl2_id = h5tools_get_fapl(fapl2_id, opts->custom_vol[1] ? &(opts->vol_info[1]) : NULL, - opts->custom_vfd[1] ? &(opts->vfd_info[1]) : NULL)) < 0) { - parallel_print("h5diff: unable to create fapl for output file\n"); - H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n"); + if (opts->page_cache > 0) { + if (H5Pset_page_buffer_size(fapl2_id, opts->page_cache, 0, 0) < 0) { + parallel_print("h5diff: unable to set page buffer cache size for fapl for output file\n"); + H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to set page buffer cache size for output fapl\n"); } } - if ((file2_id = h5tools_fopen(fname2, H5F_ACC_RDONLY, fapl2_id, (fapl2_id != H5P_DEFAULT), NULL, - (size_t)0)) < 0) { + if ((file2_id = h5tools_fopen(fname2, H5F_ACC_RDONLY, fapl2_id, + (opts->custom_vol[1] || opts->custom_vfd[1]), NULL, (size_t)0)) < 0) { parallel_print("h5diff: <%s>: unable to open file\n", fname2); H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "<%s>: unable to open file\n", fname2); } diff --git a/tools/lib/h5diff.h b/tools/lib/h5diff.h index 12bf56a8ecd..4b9847a3044 100644 --- a/tools/lib/h5diff.h +++ b/tools/lib/h5diff.h @@ -92,6 +92,7 @@ typedef struct { h5tools_vfd_info_t vfd_info[2]; /* VFD information for input file, output file */ bool custom_vol[2]; /* Using a custom input, output VOL? */ bool custom_vfd[2]; /* Using a custom input, output VFD? */ + size_t page_cache; /* Size to request for page buffer cache */ } diff_opt_t; /*------------------------------------------------------------------------- diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 16253b195d5..29419e297af 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -469,7 +469,7 @@ h5tools_set_error_file(const char *fname, int is_bin) * negative - failed *------------------------------------------------------------------------- */ -static herr_t +herr_t h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info) { herr_t ret_value = SUCCEED; @@ -650,7 +650,7 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info) * negative - failed *------------------------------------------------------------------------- */ -static herr_t +herr_t h5tools_set_fapl_vol(hid_t fapl_id, h5tools_vol_info_t *vol_info) { htri_t connector_is_registered; @@ -746,9 +746,9 @@ h5tools_set_fapl_vol(hid_t fapl_id, h5tools_vol_info_t *vol_info) } /*------------------------------------------------------------------------- - * Function: h5tools_get_fapl + * Function: h5tools_get_new_fapl * - * Purpose: Copies an input fapl and then sets a VOL and/or a VFD on it. + * Purpose: Copies an input fapl. * * The returned fapl must be closed by the caller. * @@ -757,7 +757,7 @@ h5tools_set_fapl_vol(hid_t fapl_id, h5tools_vol_info_t *vol_info) *------------------------------------------------------------------------- */ hid_t -h5tools_get_fapl(hid_t prev_fapl_id, h5tools_vol_info_t *vol_info, h5tools_vfd_info_t *vfd_info) +h5tools_get_new_fapl(hid_t prev_fapl_id) { hid_t new_fapl_id = H5I_INVALID_HID; hid_t ret_value = H5I_INVALID_HID; @@ -775,16 +775,6 @@ h5tools_get_fapl(hid_t prev_fapl_id, h5tools_vol_info_t *vol_info, h5tools_vfd_i H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "H5Pcopy failed"); } - /* Set non-default VOL connector, if requested */ - if (vol_info) - if (h5tools_set_fapl_vol(new_fapl_id, vol_info) < 0) - H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "failed to set VOL on FAPL"); - - /* Set non-default virtual file driver, if requested */ - if (vfd_info) - if (h5tools_set_fapl_vfd(new_fapl_id, vfd_info) < 0) - H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "failed to set VFD on FAPL"); - ret_value = new_fapl_id; done: @@ -1026,9 +1016,23 @@ h5tools_fopen(const char *fname, unsigned flags, hid_t fapl_id, bool use_specifi vfd_info.u.name = drivernames[drivernum]; /* Get a fapl reflecting the selected VOL connector and VFD */ - if ((tmp_fapl_id = h5tools_get_fapl(fapl_id, &vol_info, &vfd_info)) < 0) + if ((tmp_fapl_id = h5tools_get_new_fapl(fapl_id)) < 0) continue; + if (h5tools_set_fapl_vol(tmp_fapl_id, &vol_info) < 0) { + /* Close the temporary fapl */ + H5Pclose(tmp_fapl_id); + tmp_fapl_id = H5I_INVALID_HID; + continue; + } + + if (h5tools_set_fapl_vfd(tmp_fapl_id, &vfd_info) < 0) { + /* Close the temporary fapl */ + H5Pclose(tmp_fapl_id); + tmp_fapl_id = H5I_INVALID_HID; + continue; + } + /* Can we open the file with this combo? */ H5E_BEGIN_TRY { @@ -1051,9 +1055,16 @@ h5tools_fopen(const char *fname, unsigned flags, hid_t fapl_id, bool use_specifi /* NOT the native VOL connector */ /* Get a FAPL for the current VOL connector */ - if ((tmp_fapl_id = h5tools_get_fapl(fapl_id, &vol_info, NULL)) < 0) + if ((tmp_fapl_id = h5tools_get_new_fapl(fapl_id)) < 0) continue; + if (h5tools_set_fapl_vol(tmp_fapl_id, &vol_info) < 0) { + /* Close the temporary fapl */ + H5Pclose(tmp_fapl_id); + tmp_fapl_id = H5I_INVALID_HID; + continue; + } + /* Can we open the file with this connector? */ if ((fid = h5tools_fopen(fname, flags, tmp_fapl_id, true, drivername, drivername_size)) >= 0) { used_fapl_id = tmp_fapl_id; diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h index b4ace197d7f..52e3a3659a8 100644 --- a/tools/lib/h5tools.h +++ b/tools/lib/h5tools.h @@ -673,9 +673,10 @@ H5TOOLS_DLL int h5tools_set_input_file(const char *fname, int is_bin); H5TOOLS_DLL int h5tools_set_output_file(const char *fname, int is_bin); H5TOOLS_DLL int h5tools_set_error_file(const char *fname, int is_bin); -H5TOOLS_DLL hid_t h5tools_get_fapl(hid_t prev_fapl_id, h5tools_vol_info_t *vol_info, - h5tools_vfd_info_t *vfd_info); +H5TOOLS_DLL hid_t h5tools_get_new_fapl(hid_t prev_fapl_id); H5TOOLS_DLL herr_t h5tools_get_vfd_name(hid_t fid, hid_t fapl_id, char *drivername, size_t drivername_size); +H5TOOLS_DLL herr_t h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info); +H5TOOLS_DLL herr_t h5tools_set_fapl_vol(hid_t fapl_id, h5tools_vol_info_t *vol_info); H5TOOLS_DLL hid_t h5tools_fopen(const char *fname, unsigned flags, hid_t fapl, bool use_specific_driver, char *drivername, size_t drivername_size); H5TOOLS_DLL hid_t h5tools_get_little_endian_type(hid_t type); diff --git a/tools/libtest/h5tools_test_utils.c b/tools/libtest/h5tools_test_utils.c index 369472e8d86..03c595f66cb 100644 --- a/tools/libtest/h5tools_test_utils.c +++ b/tools/libtest/h5tools_test_utils.c @@ -927,7 +927,7 @@ test_populate_ros3_fa(void) * * Function: test_set_configured_fapl() * - * Purpose: Verify `h5tools_get_fapl()` with ROS3 and HDFS VFDs + * Purpose: Verify `h5tools_get_new_fapl()` with ROS3 and HDFS VFDs * * Return: 0 if test passes * 1 if failure @@ -1111,12 +1111,15 @@ test_set_configured_fapl(void) vfd_info.info = C.conf_fa; vfd_info.u.name = C.vfdname; - if (C.expected == 1) - result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info); + if (C.expected == 1) { + result = h5tools_get_new_fapl(H5P_DEFAULT); + result = h5tools_set_fapl_vfd(result, &vfd_info); + } else { H5E_BEGIN_TRY { - result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info); + result = h5tools_get_new_fapl(H5P_DEFAULT); + result = h5tools_set_fapl_vfd(result, &vfd_info); } H5E_END_TRY } diff --git a/tools/src/h5diff/h5diff_common.c b/tools/src/h5diff/h5diff_common.c index 77b83533409..62a8cd2abe5 100644 --- a/tools/src/h5diff/h5diff_common.c +++ b/tools/src/h5diff/h5diff_common.c @@ -24,7 +24,7 @@ static int check_d_input(const char *); * Command-line options: The user can specify short or long-named * parameters. */ -static const char *s_opts = "cd:ehln:p:qrv*xA:CE:NS*V"; +static const char *s_opts = "cd:ehln:p:qrv*xA:CE:K:NS*V"; static struct h5_long_options l_opts[] = {{"compare", no_arg, 'c'}, {"delta", require_arg, 'd'}, {"use-system-epsilon", no_arg, 'e'}, @@ -39,6 +39,7 @@ static struct h5_long_options l_opts[] = {{"compare", no_arg, 'c'}, {"exclude-attribute", require_arg, 'A'}, {"no-compact-subset", no_arg, 'C'}, {"exclude-path", require_arg, 'E'}, + {"page-buffer-size", require_arg, 'K'}, {"nan", no_arg, 'N'}, {"enable-error-stack", optional_arg, 'S'}, {"version", no_arg, 'V'}, @@ -137,6 +138,9 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const /**this is bad in mixing option with results**/ opts->not_cmp = 0; + /* init for page buffer cache size option */ + opts->page_cache = 0; + /* init for exclude-path option */ exclude_head = NULL; @@ -320,6 +324,10 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const opts->use_system_epsilon = 1; break; + case 'K': + opts->page_cache = strtoul(H5_optarg, NULL, 0); + break; + case '1': opts->vol_info[0].type = VOL_BY_VALUE; opts->vol_info[0].u.value = (H5VL_class_value_t)atoi(H5_optarg); @@ -622,6 +630,8 @@ usage(void) PRINTVALSTREAM(rawoutstream, " 3 : All level 2 information plus file names.\n"); PRINTVALSTREAM(rawoutstream, " -q, --quiet\n"); PRINTVALSTREAM(rawoutstream, " Quiet mode. Do not produce output.\n"); + PRINTVALSTREAM(rawoutstream, + " --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n"); PRINTVALSTREAM(rawoutstream, " --vol-value-1 Value (ID) of the VOL connector to use for opening the\n"); PRINTVALSTREAM(rawoutstream, " first HDF5 file specified\n"); diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index 40041e3aec9..dc86e526294 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -17,6 +17,7 @@ /* Name of tool */ #define PROGRAMNAME "h5dump" +size_t page_cache = 0; const char *outfname_g = NULL; static bool doxml_g = false; static bool useschema_g = true; @@ -97,7 +98,7 @@ struct handler_t { */ /* The following initialization makes use of C language concatenating */ /* "xxx" "yyy" into "xxxyyy". */ -static const char *s_opts = "a:b*c:d:ef:g:hik:l:m:n*o*pq:rs:t:uvw:xyz:A*BCD:E*F:G:HM:N:O*RS:VX:"; +static const char *s_opts = "a:b*c:d:ef:g:hik:l:m:n*o*pq:rs:t:uvw:xyz:A*BCD:E*F:G:HK:M:N:O*RS:VX:"; static struct h5_long_options l_opts[] = {{"attribute", require_arg, 'a'}, {"binary", optional_arg, 'b'}, {"count", require_arg, 'c'}, @@ -132,6 +133,7 @@ static struct h5_long_options l_opts[] = {{"attribute", require_arg, 'a'}, {"form", require_arg, 'F'}, {"vds-gap-size", require_arg, 'G'}, {"header", no_arg, 'H'}, + {"page-buffer-size", require_arg, 'K'}, {"packed-bits", require_arg, 'M'}, {"any_path", require_arg, 'N'}, {"ddl", optional_arg, 'O'}, @@ -198,6 +200,8 @@ usage(const char *prog) PRINTVALSTREAM(rawoutstream, " -O F, --ddl=F Output ddl text into file F\n"); PRINTVALSTREAM(rawoutstream, " Use blank(empty) filename F to suppress ddl display\n"); + PRINTVALSTREAM(rawoutstream, + " --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n"); PRINTVALSTREAM(rawoutstream, " --s3-cred= Supply S3 authentication information to \"ros3\" vfd.\n"); PRINTVALSTREAM(rawoutstream, @@ -1019,6 +1023,9 @@ parse_command_line(int argc, const char *const *argv) goto error; } break; + case 'K': + page_cache = strtoul(H5_optarg, NULL, 0); + break; /** begin XML parameters **/ case 'x': @@ -1368,10 +1375,30 @@ main(int argc, char *argv[]) /* Initialize indexing options */ h5trav_set_index(sort_by, sort_order); - if (use_custom_vol_g || use_custom_vfd_g) { - if ((fapl_id = h5tools_get_fapl(H5P_DEFAULT, use_custom_vol_g ? &vol_info_g : NULL, - use_custom_vfd_g ? &vfd_info_g : NULL)) < 0) { - error_msg("unable to create FAPL for file access\n"); + if ((fapl_id = h5tools_get_new_fapl(H5P_DEFAULT)) < 0) { + error_msg("unable to create FAPL for file access\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + /* Set non-default VOL connector, if requested */ + if (use_custom_vol_g) { + if (h5tools_set_fapl_vol(fapl_id, &vol_info_g) < 0) { + error_msg("unable to set VOL on fapl for file\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + } + /* Set non-default virtual file driver, if requested */ + if (use_custom_vfd_g) { + if (h5tools_set_fapl_vfd(fapl_id, &vfd_info_g) < 0) { + error_msg("unable to set VFD on fapl for file\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + } + if (page_cache > 0) { + if (H5Pset_page_buffer_size(fapl_id, page_cache, 0, 0) < 0) { + error_msg("unable to set page buffer cache size for file access\n"); h5tools_setstatus(EXIT_FAILURE); goto done; } @@ -1394,7 +1421,8 @@ main(int argc, char *argv[]) goto done; } else - fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0); + fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (use_custom_vol_g || use_custom_vfd_g), NULL, + 0); if (fid < 0) { error_msg("unable to open file \"%s\"\n", fname); diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c index ac3835564bf..3e9255019ca 100644 --- a/tools/src/h5ls/h5ls.c +++ b/tools/src/h5ls/h5ls.c @@ -25,6 +25,7 @@ #define PROGRAMNAME "h5ls" #define NAME_BUF_SIZE 2048 + /* * Alternative formatting for data dumped by H5LS * @@ -211,6 +212,8 @@ usage(void) PRINTVALSTREAM(rawoutstream, " -wN, --width=N Set the number of columns of output\n"); PRINTVALSTREAM(rawoutstream, " -v, --verbose Generate more verbose output\n"); PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n"); + PRINTVALSTREAM(rawoutstream, + " --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n"); PRINTVALSTREAM(rawoutstream, " --vfd=DRIVER Use the specified virtual file driver\n"); PRINTVALSTREAM(rawoutstream, " -x, --hexdump Show raw data in hexadecimal format\n"); PRINTVALSTREAM(rawoutstream, @@ -2659,6 +2662,7 @@ main(int argc, char *argv[]) static char root_name[] = "/"; char drivername[50]; int err_exit = 0; + size_t page_cache = 0; hid_t fapl_id = H5P_DEFAULT; bool custom_vol_fapl = false; bool custom_vfd_fapl = false; @@ -2816,6 +2820,9 @@ main(int argc, char *argv[]) leave(EXIT_FAILURE); } } + else if (!strncmp(argv[argno], "--page-buffer-size=", (size_t)19)) { + page_cache = strtoul(argv[argno] + 19, &rest, 0); + } else if (!strcmp(argv[argno], "--verbose")) { verbose_g++; } @@ -2995,23 +3002,40 @@ main(int argc, char *argv[]) } /* Setup a custom fapl for file accesses */ - if (custom_vol_fapl || custom_vfd_fapl) { #ifdef H5_HAVE_ROS3_VFD - if (custom_vfd_fapl && (0 == strcmp(vfd_info.u.name, drivernames[ROS3_VFD_IDX]))) { - if (!vfd_info.info) - vfd_info.info = &ros3_fa; - } + if (custom_vfd_fapl && (0 == strcmp(vfd_info.u.name, drivernames[ROS3_VFD_IDX]))) { + if (!vfd_info.info) + vfd_info.info = &ros3_fa; + } #endif #ifdef H5_HAVE_LIBHDFS - if (custom_vfd_fapl && (0 == strcmp(vfd_info.u.name, drivernames[HDFS_VFD_IDX]))) { - if (!vfd_info.info) - vfd_info.info = &hdfs_fa; - } + if (custom_vfd_fapl && (0 == strcmp(vfd_info.u.name, drivernames[HDFS_VFD_IDX]))) { + if (!vfd_info.info) + vfd_info.info = &hdfs_fa; + } #endif - if ((fapl_id = h5tools_get_fapl(H5P_DEFAULT, custom_vol_fapl ? &vol_info : NULL, - custom_vfd_fapl ? &vfd_info : NULL)) < 0) { - error_msg("failed to setup file access property list (fapl) for file\n"); + if ((fapl_id = h5tools_get_new_fapl(H5P_DEFAULT)) < 0) { + error_msg("unable to create FAPL for file access\n"); + leave(EXIT_FAILURE); + } + /* Set non-default VOL connector, if requested */ + if (custom_vol_fapl) { + if (h5tools_set_fapl_vol(fapl_id, &vol_info) < 0) { + error_msg("unable to set VOL on fapl for file\n"); + leave(EXIT_FAILURE); + } + } + /* Set non-default virtual file driver, if requested */ + if (custom_vfd_fapl) { + if (h5tools_set_fapl_vfd(fapl_id, &vfd_info) < 0) { + error_msg("unable to set VFD on fapl for file\n"); + leave(EXIT_FAILURE); + } + } + if (page_cache > 0) { + if (H5Pset_page_buffer_size(fapl_id, page_cache, 0, 0) < 0) { + error_msg("unable to set page buffer cache size for file access\n"); leave(EXIT_FAILURE); } } @@ -3040,8 +3064,8 @@ main(int argc, char *argv[]) file_id = H5I_INVALID_HID; while (fname && *fname) { - file_id = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), drivername, - sizeof drivername); + file_id = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (custom_vol_fapl || custom_vfd_fapl), + drivername, sizeof drivername); if (file_id >= 0) { if (verbose_g) diff --git a/tools/src/h5repack/h5repack.c b/tools/src/h5repack/h5repack.c index 30351e220d7..fa1dc90081b 100644 --- a/tools/src/h5repack/h5repack.c +++ b/tools/src/h5repack/h5repack.c @@ -80,6 +80,10 @@ h5repack_init(pack_opt_t *options, int verbose, bool latest) options->high_bound = H5F_LIBVER_LATEST; options->fin_fapl = H5P_DEFAULT; options->fout_fapl = H5P_DEFAULT; + options->fin_vol = false; + options->fin_vfd = false; + options->fout_vol = false; + options->fout_vfd = false; for (n = 0; n < H5_REPACK_MAX_NFILTERS; n++) { options->filter_g[n].filtn = -1; @@ -793,7 +797,7 @@ check_objects(const char *fname, pack_opt_t *options) * open the file *------------------------------------------------------------------------- */ - if ((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT), + if ((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_vol || options->fin_vfd), NULL, 0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname, H5FOPENERROR); diff --git a/tools/src/h5repack/h5repack.h b/tools/src/h5repack/h5repack.h index fa79c6f53f5..cc724a067cf 100644 --- a/tools/src/h5repack/h5repack.h +++ b/tools/src/h5repack/h5repack.h @@ -133,6 +133,10 @@ typedef struct { int fs_persist; /* Free space section threshold */ long fs_threshold; /* Free space section threshold */ long long fs_pagesize; /* File space page size */ + bool fin_vol; /* Custom VOL for input file */ + bool fin_vfd; /* Custom VFD for input file */ + bool fout_vol; /* Custom VOL for output file */ + bool fout_vfd; /* Custom VFD for output file */ } pack_opt_t; typedef struct named_dt_t { @@ -156,7 +160,7 @@ int h5repack_addlayout(const char *str, pack_opt_t *options); int h5repack_init(pack_opt_t *options, int verbose, bool latest); int h5repack_end(pack_opt_t *options); int h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options); -int h5repack_cmp_pl(const char *fname1, hid_t fname1_fapl, const char *fname2, hid_t fname2_fapl); +int h5repack_cmp_pl(const char *fname1, const char *fname2, pack_opt_t *options); /* Note: The below copy_named_datatype(), named_datatype_free(), copy_attr() * and struct named_dt_t were located in h5repack_copy.c as static prior to diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index ca1e76753d4..77442269f6f 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -81,8 +81,8 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) * open input file *------------------------------------------------------------------------- */ - if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT), - NULL, (size_t)0)) < 0) + if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, + (options->fin_vol || options->fin_vfd), NULL, (size_t)0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fnamein, H5FOPENERROR); /* get user block size and file space strategy/persist/threshold */ @@ -849,7 +849,7 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti * doesn't support this. */ if (use_h5ocopy && - (options->fin_fapl != H5P_DEFAULT || options->fout_fapl != H5P_DEFAULT)) { + (options->fin_vol || options->fout_vol || options->fin_vfd || options->fout_vfd)) { hid_t in_vol_id; hid_t out_vol_id; hid_t default_vol_id; diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c index afe0bc953c7..9fb85f13cdd 100644 --- a/tools/src/h5repack/h5repack_main.c +++ b/tools/src/h5repack/h5repack_main.c @@ -30,7 +30,8 @@ static const char *outfile = NULL; * Command-line options: The user can specify short or long-named * parameters. */ -static const char *s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:v*z:E*G:LM:P:S:T:VXWY:Z:1:2:3:4:5:6:7:8:9:0:"; +static const char *s_opts = + "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:v*z:E*G:K:LM:P:S:T:VXWY:Z:1:2:3:4:5:6:7:8:9:0:"; static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'}, {"block", require_arg, 'b'}, {"compact", require_arg, 'c'}, @@ -53,6 +54,7 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'}, {"sort_order", require_arg, 'z'}, {"enable-error-stack", optional_arg, 'E'}, {"fs_pagesize", require_arg, 'G'}, + {"page-buffer-size", require_arg, 'K'}, {"latest", no_arg, 'L'}, {"metadata_block_size", require_arg, 'M'}, {"fs_persist", require_arg, 'P'}, @@ -113,6 +115,8 @@ usage(const char *prog) PRINTVALSTREAM(rawoutstream, " N - is an integer greater than 1, 2 displays read/write timing\n"); PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n"); PRINTVALSTREAM(rawoutstream, " -n, --native Use a native HDF5 type when repacking\n"); + PRINTVALSTREAM(rawoutstream, + " --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n"); PRINTVALSTREAM(rawoutstream, " --src-vol-value Value (ID) of the VOL connector to use for opening the\n"); PRINTVALSTREAM(rawoutstream, " input HDF5 file specified\n"); @@ -526,11 +530,8 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) h5tools_vol_info_t out_vol_info; h5tools_vfd_info_t in_vfd_info; h5tools_vfd_info_t out_vfd_info; - bool custom_in_vol = false; - bool custom_in_vfd = false; - bool custom_out_vol = false; - bool custom_out_vfd = false; - hid_t tmp_fapl = H5I_INVALID_HID; + hid_t tmp_fapl = H5I_INVALID_HID; + size_t page_cache = 0; int bound, opt; int ret_value = 0; @@ -765,6 +766,10 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) options->fs_pagesize = -1; break; + case 'K': + page_cache = strtoul(H5_optarg, NULL, 0); + break; + case 'q': if (H5_INDEX_UNKNOWN == (sort_by = set_sort_by(H5_optarg))) { error_msg("failed to set sort by form <%s>\n", H5_optarg); @@ -793,13 +798,13 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) case '1': in_vol_info.type = VOL_BY_VALUE; in_vol_info.u.value = (H5VL_class_value_t)atoi(H5_optarg); - custom_in_vol = true; + options->fin_vol = true; break; case '2': in_vol_info.type = VOL_BY_NAME; in_vol_info.u.name = H5_optarg; - custom_in_vol = true; + options->fin_vol = true; break; case '3': @@ -809,13 +814,13 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) case '4': out_vol_info.type = VOL_BY_VALUE; out_vol_info.u.value = (H5VL_class_value_t)atoi(H5_optarg); - custom_out_vol = true; + options->fout_vol = true; break; case '5': out_vol_info.type = VOL_BY_NAME; out_vol_info.u.name = H5_optarg; - custom_out_vol = true; + options->fout_vol = true; break; case '6': @@ -825,13 +830,13 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) case '7': in_vfd_info.type = VFD_BY_VALUE; in_vfd_info.u.value = (H5FD_class_value_t)atoi(H5_optarg); - custom_in_vfd = true; + options->fin_vfd = true; break; case '8': in_vfd_info.type = VFD_BY_NAME; in_vfd_info.u.name = H5_optarg; - custom_in_vfd = true; + options->fin_vfd = true; break; case '9': @@ -841,13 +846,13 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) case '0': out_vfd_info.type = VFD_BY_VALUE; out_vfd_info.u.value = (H5FD_class_value_t)atoi(H5_optarg); - custom_out_vfd = true; + options->fout_vfd = true; break; case 'Y': out_vfd_info.type = VFD_BY_NAME; out_vfd_info.u.name = H5_optarg; - custom_out_vfd = true; + options->fout_vfd = true; break; case 'Z': @@ -904,47 +909,68 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) } /* Setup FAPL for input and output file accesses */ - if (custom_in_vol || custom_in_vfd) { - if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, custom_in_vol ? &in_vol_info : NULL, - custom_in_vfd ? &in_vfd_info : NULL)) < 0) { - error_msg("failed to setup FAPL for input file\n"); + if ((tmp_fapl = h5tools_get_new_fapl(options->fin_fapl)) < 0) { + error_msg("unable to create FAPL for file access for input file\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + goto done; + } + /* Set non-default VOL connector, if requested */ + if (options->fin_vol) { + if (h5tools_set_fapl_vol(tmp_fapl, &in_vol_info) < 0) { + error_msg("unable to set VOL on fapl for input file\n"); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; } - - /* Close old FAPL */ - if (options->fin_fapl != H5P_DEFAULT) - if (H5Pclose(options->fin_fapl) < 0) { - error_msg("failed to close FAPL\n"); - h5tools_setstatus(EXIT_FAILURE); - ret_value = -1; - goto done; - } - - options->fin_fapl = tmp_fapl; } - - if (custom_out_vol || custom_out_vfd) { - if ((tmp_fapl = h5tools_get_fapl(options->fout_fapl, custom_out_vol ? &out_vol_info : NULL, - custom_out_vfd ? &out_vfd_info : NULL)) < 0) { - error_msg("failed to setup FAPL for output file\n"); + /* Set non-default virtual file driver, if requested */ + if (options->fin_vfd) { + if (h5tools_set_fapl_vfd(tmp_fapl, &in_vfd_info) < 0) { + error_msg("unable to set VFD on fapl for input file\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + goto done; + } + } + if (page_cache > 0) { + if (H5Pset_page_buffer_size(tmp_fapl, page_cache, 0, 0) < 0) { + error_msg("unable to set page buffer cache size for input file\n"); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; } + } - /* Close old FAPL */ - if (options->fout_fapl != H5P_DEFAULT) - if (H5Pclose(options->fout_fapl) < 0) { - error_msg("failed to close FAPL\n"); - h5tools_setstatus(EXIT_FAILURE); - ret_value = -1; - goto done; - } + options->fin_fapl = tmp_fapl; - options->fout_fapl = tmp_fapl; + /* Setup FAPL for input and output file accesses */ + if ((tmp_fapl = h5tools_get_new_fapl(options->fout_fapl)) < 0) { + error_msg("unable to create FAPL for file access for output file\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + goto done; } + /* Set non-default VOL connector, if requested */ + if (options->fout_vol) { + if (h5tools_set_fapl_vol(tmp_fapl, &out_vol_info) < 0) { + error_msg("unable to set VOL on fapl for output file\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + goto done; + } + } + /* Set non-default virtual file driver, if requested */ + if (options->fout_vfd) { + if (h5tools_set_fapl_vfd(tmp_fapl, &out_vfd_info) < 0) { + error_msg("unable to set VFD on fapl for output file\n"); + h5tools_setstatus(EXIT_FAILURE); + ret_value = -1; + goto done; + } + } + + options->fout_fapl = tmp_fapl; done: return ret_value; diff --git a/tools/src/h5repack/h5repack_verify.c b/tools/src/h5repack/h5repack_verify.c index 819dd94ab26..50604e055cb 100644 --- a/tools/src/h5repack/h5repack_verify.c +++ b/tools/src/h5repack/h5repack_verify.c @@ -356,7 +356,7 @@ verify_layout(hid_t pid, pack_info_t *obj) */ int -h5repack_cmp_pl(const char *fname1, hid_t fname1_fapl, const char *fname2, hid_t fname2_fapl) +h5repack_cmp_pl(const char *fname1, const char *fname2, pack_opt_t *options) { hid_t fid1 = H5I_INVALID_HID; /* file ID */ hid_t fid2 = H5I_INVALID_HID; /* file ID */ @@ -377,11 +377,11 @@ h5repack_cmp_pl(const char *fname1, hid_t fname1_fapl, const char *fname2, hid_t *------------------------------------------------------------------------- */ /* Open the files */ - if ((fid1 = h5tools_fopen(fname1, H5F_ACC_RDONLY, fname1_fapl, (fname1_fapl != H5P_DEFAULT), NULL, 0)) < - 0) + if ((fid1 = h5tools_fopen(fname1, H5F_ACC_RDONLY, options->fin_fapl, + (options->fin_vol || options->fin_vfd), NULL, 0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname1, H5FOPENERROR); - if ((fid2 = h5tools_fopen(fname2, H5F_ACC_RDONLY, fname2_fapl, (fname2_fapl != H5P_DEFAULT), NULL, 0)) < - 0) + if ((fid2 = h5tools_fopen(fname2, H5F_ACC_RDONLY, options->fout_fapl, + (options->fout_vol || options->fout_vfd), NULL, 0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname2, H5FOPENERROR); /*------------------------------------------------------------------------- diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index a90ef5e5e8f..71c507801ee 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -117,6 +117,8 @@ typedef struct iter_t { static const char *drivername = NULL; +size_t page_cache = 0; + #ifdef H5_HAVE_ROS3_VFD /* Default "anonymous" S3 configuration */ static H5FD_ros3_fapl_ext_t ros3_fa = { @@ -171,7 +173,7 @@ struct handler_t { char **obj; }; -static const char *s_opts = "Aa:Ddm:E*FfhGgl:sSTO:Vw:H:"; +static const char *s_opts = "Aa:Ddm:E*FfhGgl:K:sSTO:Vw:H:"; /* e.g. "filemetadata" has to precede "file"; "groupmetadata" has to precede "group" etc. */ static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"filemetadata", no_arg, 'F'}, @@ -187,6 +189,7 @@ static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"numattrs", require_arg, 'a'}, {"freespace", no_arg, 's'}, {"summary", no_arg, 'S'}, + {"page-buffer-size", require_arg, 'K'}, {"s3-cred", require_arg, 'w'}, {"hdfs-attrs", require_arg, 'H'}, {NULL, 0, '\0'}}; @@ -239,6 +242,7 @@ usage(const char *prog) fprintf(stdout, " than 0. The default threshold is 10.\n"); fprintf(stdout, " -s, --freespace Print free space information\n"); fprintf(stdout, " -S, --summary Print summary of file space information\n"); + fprintf(stdout, " --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n"); fprintf(stdout, " --s3-cred= Access file on S3, using provided credential\n"); fprintf(stdout, " :: (region,id,key)\n"); fprintf(stdout, " If == \"(,,)\", no authentication is used.\n"); @@ -967,6 +971,10 @@ parse_command_line(int argc, const char *const *argv, struct handler_t **hand_re #endif break; + case 'K': + page_cache = strtoul(H5_optarg, NULL, 0); + break; + default: usage(h5tools_getprogname()); goto error; @@ -1618,6 +1626,11 @@ main(int argc, char *argv[]) /* enable error reporting if command line option */ h5tools_error_report(); + if ((fapl_id = h5tools_get_new_fapl(H5P_DEFAULT)) < 0) { + error_msg("unable to create FAPL for file access\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } if (drivername) { h5tools_vfd_info_t vfd_info; @@ -1634,8 +1647,17 @@ main(int argc, char *argv[]) vfd_info.info = &hdfs_fa; #endif - if ((fapl_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info)) < 0) { - error_msg("Unable to create FAPL for file access\n"); + /* Set non-default virtual file driver, if requested */ + if (h5tools_set_fapl_vfd(fapl_id, &vfd_info) < 0) { + error_msg("unable to set VFD on fapl for file\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + } + if (page_cache > 0) { + if (H5Pset_page_buffer_size(fapl_id, page_cache, 0, 0) < 0) { + error_msg("unable to set page buffer cache size for file access\n"); + h5tools_setstatus(EXIT_FAILURE); goto done; } } @@ -1649,7 +1671,7 @@ main(int argc, char *argv[]) printf("Filename: %s\n", fname); - fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0); + fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (drivername != NULL), NULL, 0); if (fid < 0) { error_msg("unable to open file \"%s\"\n", fname); diff --git a/tools/src/misc/h5mkgrp.c b/tools/src/misc/h5mkgrp.c index 92e30423661..b96251991b5 100644 --- a/tools/src/misc/h5mkgrp.c +++ b/tools/src/misc/h5mkgrp.c @@ -30,13 +30,15 @@ static struct h5_long_options l_opts[] = { /* Command line parameter settings */ typedef struct mkgrp_opt_t { - char *fname; /* File name to operate on */ - bool latest; /* Whether file should use latest format versions */ - bool verbose; /* Whether output should be verbose */ - bool parents; /* Whether to create intermediate groups */ - size_t ngroups; /* Number of groups to create */ - char **groups; /* Pointer to array of group names */ - hid_t fapl_id; /* fapl to use when opening the file */ + char *fname; /* File name to operate on */ + bool latest; /* Whether file should use latest format versions */ + bool verbose; /* Whether output should be verbose */ + bool parents; /* Whether to create intermediate groups */ + size_t ngroups; /* Number of groups to create */ + char **groups; /* Pointer to array of group names */ + hid_t fapl_id; /* fapl to use when opening the file */ + bool custom_vol; /* Whether fapl uses custom VOL */ + bool custom_vfd; /* Whether fapl uses custom VFD */ } mkgrp_opt_t; static mkgrp_opt_t params_g; /* Command line parameter settings */ @@ -135,8 +137,6 @@ parse_command_line(int argc, const char *const *argv, mkgrp_opt_t *options) { int opt; /* Option from command line */ size_t curr_group; /* Current group name to copy */ - bool custom_vol = false; - bool custom_vfd = false; h5tools_vol_info_t vol_info; h5tools_vfd_info_t vfd_info; hid_t tmp_fapl_id = H5I_INVALID_HID; @@ -182,15 +182,15 @@ parse_command_line(int argc, const char *const *argv, mkgrp_opt_t *options) break; case '1': - vol_info.type = VOL_BY_VALUE; - vol_info.u.value = (H5VL_class_value_t)atoi(H5_optarg); - custom_vol = true; + vol_info.type = VOL_BY_VALUE; + vol_info.u.value = (H5VL_class_value_t)atoi(H5_optarg); + options->custom_vol = true; break; case '2': - vol_info.type = VOL_BY_NAME; - vol_info.u.name = H5_optarg; - custom_vol = true; + vol_info.type = VOL_BY_NAME; + vol_info.u.name = H5_optarg; + options->custom_vol = true; break; case '3': @@ -198,15 +198,15 @@ parse_command_line(int argc, const char *const *argv, mkgrp_opt_t *options) break; case '4': - vfd_info.type = VFD_BY_VALUE; - vfd_info.u.value = (H5FD_class_value_t)atoi(H5_optarg); - custom_vfd = true; + vfd_info.type = VFD_BY_VALUE; + vfd_info.u.value = (H5FD_class_value_t)atoi(H5_optarg); + options->custom_vfd = true; break; case '5': - vfd_info.type = VFD_BY_NAME; - vfd_info.u.name = H5_optarg; - custom_vfd = true; + vfd_info.type = VFD_BY_NAME; + vfd_info.u.name = H5_optarg; + options->custom_vfd = true; break; case '6': @@ -251,19 +251,25 @@ parse_command_line(int argc, const char *const *argv, mkgrp_opt_t *options) } /* Setup a custom fapl for file accesses */ - if (custom_vol || custom_vfd) { - if ((tmp_fapl_id = h5tools_get_fapl(options->fapl_id, custom_vol ? &vol_info : NULL, - custom_vfd ? &vfd_info : NULL)) < 0) { - error_msg("failed to setup file access property list (fapl) for file\n"); + if (options->custom_vol || options->custom_vfd) { + if ((tmp_fapl_id = h5tools_get_new_fapl(options->fapl_id)) < 0) { + error_msg("unable to create FAPL for file access\n"); leave(EXIT_FAILURE); } - - /* Close the old fapl */ - if (options->fapl_id != H5P_DEFAULT) - if (H5Pclose(options->fapl_id) < 0) { - error_msg("failed to close file access property list (fapl)\n"); + /* Set non-default VOL connector, if requested */ + if (options->custom_vol) { + if (h5tools_set_fapl_vol(tmp_fapl_id, &vol_info) < 0) { + error_msg("unable to set VOL on fapl for file\n"); leave(EXIT_FAILURE); } + } + /* Set non-default virtual file driver, if requested */ + if (options->custom_vfd) { + if (h5tools_set_fapl_vfd(tmp_fapl_id, &vfd_info) < 0) { + error_msg("unable to set VFD on fapl for file\n"); + leave(EXIT_FAILURE); + } + } options->fapl_id = tmp_fapl_id; } @@ -323,8 +329,8 @@ main(int argc, char *argv[]) } /* Attempt to open an existing HDF5 file first */ - fid = h5tools_fopen(params_g.fname, H5F_ACC_RDWR, params_g.fapl_id, (params_g.fapl_id != H5P_DEFAULT), - NULL, 0); + fid = h5tools_fopen(params_g.fname, H5F_ACC_RDWR, params_g.fapl_id, + (params_g.custom_vol || params_g.custom_vfd), NULL, 0); /* If we couldn't open an existing file, try creating file */ /* (use "EXCL" instead of "TRUNC", so we don't blow away existing non-HDF5 file) */ diff --git a/tools/test/h5diff/expected/h5diff_10.txt b/tools/test/h5diff/expected/h5diff_10.txt index a30da23a8fe..30a62550fd9 100644 --- a/tools/test/h5diff/expected/h5diff_10.txt +++ b/tools/test/h5diff/expected/h5diff_10.txt @@ -25,6 +25,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] 3 : All level 2 information plus file names. -q, --quiet Quiet mode. Do not produce output. + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vol-value-1 Value (ID) of the VOL connector to use for opening the first HDF5 file specified --vol-name-1 Name of the VOL connector to use for opening the first diff --git a/tools/test/h5diff/expected/h5diff_600.txt b/tools/test/h5diff/expected/h5diff_600.txt index bdb7788af7d..5a081f96b3a 100644 --- a/tools/test/h5diff/expected/h5diff_600.txt +++ b/tools/test/h5diff/expected/h5diff_600.txt @@ -25,6 +25,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] 3 : All level 2 information plus file names. -q, --quiet Quiet mode. Do not produce output. + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vol-value-1 Value (ID) of the VOL connector to use for opening the first HDF5 file specified --vol-name-1 Name of the VOL connector to use for opening the first diff --git a/tools/test/h5diff/expected/h5diff_603.txt b/tools/test/h5diff/expected/h5diff_603.txt index ce1577c5f82..12e67ddb02b 100644 --- a/tools/test/h5diff/expected/h5diff_603.txt +++ b/tools/test/h5diff/expected/h5diff_603.txt @@ -26,6 +26,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] 3 : All level 2 information plus file names. -q, --quiet Quiet mode. Do not produce output. + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vol-value-1 Value (ID) of the VOL connector to use for opening the first HDF5 file specified --vol-name-1 Name of the VOL connector to use for opening the first diff --git a/tools/test/h5diff/expected/h5diff_606.txt b/tools/test/h5diff/expected/h5diff_606.txt index 9ee22448c5b..1d04cb92f53 100644 --- a/tools/test/h5diff/expected/h5diff_606.txt +++ b/tools/test/h5diff/expected/h5diff_606.txt @@ -26,6 +26,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] 3 : All level 2 information plus file names. -q, --quiet Quiet mode. Do not produce output. + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vol-value-1 Value (ID) of the VOL connector to use for opening the first HDF5 file specified --vol-name-1 Name of the VOL connector to use for opening the first diff --git a/tools/test/h5diff/expected/h5diff_612.txt b/tools/test/h5diff/expected/h5diff_612.txt index fee63aaed28..130970a1f45 100644 --- a/tools/test/h5diff/expected/h5diff_612.txt +++ b/tools/test/h5diff/expected/h5diff_612.txt @@ -26,6 +26,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] 3 : All level 2 information plus file names. -q, --quiet Quiet mode. Do not produce output. + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vol-value-1 Value (ID) of the VOL connector to use for opening the first HDF5 file specified --vol-name-1 Name of the VOL connector to use for opening the first diff --git a/tools/test/h5diff/expected/h5diff_615.txt b/tools/test/h5diff/expected/h5diff_615.txt index 562308aaa9a..9c086bf2360 100644 --- a/tools/test/h5diff/expected/h5diff_615.txt +++ b/tools/test/h5diff/expected/h5diff_615.txt @@ -26,6 +26,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] 3 : All level 2 information plus file names. -q, --quiet Quiet mode. Do not produce output. + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vol-value-1 Value (ID) of the VOL connector to use for opening the first HDF5 file specified --vol-name-1 Name of the VOL connector to use for opening the first diff --git a/tools/test/h5diff/expected/h5diff_621.txt b/tools/test/h5diff/expected/h5diff_621.txt index 03b69bba232..c76aae8b347 100644 --- a/tools/test/h5diff/expected/h5diff_621.txt +++ b/tools/test/h5diff/expected/h5diff_621.txt @@ -26,6 +26,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] 3 : All level 2 information plus file names. -q, --quiet Quiet mode. Do not produce output. + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vol-value-1 Value (ID) of the VOL connector to use for opening the first HDF5 file specified --vol-name-1 Name of the VOL connector to use for opening the first diff --git a/tools/test/h5diff/expected/h5diff_622.txt b/tools/test/h5diff/expected/h5diff_622.txt index 05e6b94c502..99f396aa40d 100644 --- a/tools/test/h5diff/expected/h5diff_622.txt +++ b/tools/test/h5diff/expected/h5diff_622.txt @@ -26,6 +26,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] 3 : All level 2 information plus file names. -q, --quiet Quiet mode. Do not produce output. + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vol-value-1 Value (ID) of the VOL connector to use for opening the first HDF5 file specified --vol-name-1 Name of the VOL connector to use for opening the first diff --git a/tools/test/h5diff/expected/h5diff_623.txt b/tools/test/h5diff/expected/h5diff_623.txt index d2e81fa8f47..3490630de93 100644 --- a/tools/test/h5diff/expected/h5diff_623.txt +++ b/tools/test/h5diff/expected/h5diff_623.txt @@ -26,6 +26,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] 3 : All level 2 information plus file names. -q, --quiet Quiet mode. Do not produce output. + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vol-value-1 Value (ID) of the VOL connector to use for opening the first HDF5 file specified --vol-name-1 Name of the VOL connector to use for opening the first diff --git a/tools/test/h5diff/expected/h5diff_624.txt b/tools/test/h5diff/expected/h5diff_624.txt index 94e6b4ef8af..f6ff568800e 100644 --- a/tools/test/h5diff/expected/h5diff_624.txt +++ b/tools/test/h5diff/expected/h5diff_624.txt @@ -26,6 +26,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] 3 : All level 2 information plus file names. -q, --quiet Quiet mode. Do not produce output. + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vol-value-1 Value (ID) of the VOL connector to use for opening the first HDF5 file specified --vol-name-1 Name of the VOL connector to use for opening the first diff --git a/tools/test/h5dump/CMakeTests.cmake b/tools/test/h5dump/CMakeTests.cmake index 546cfcbf011..05f3648eb57 100644 --- a/tools/test/h5dump/CMakeTests.cmake +++ b/tools/test/h5dump/CMakeTests.cmake @@ -23,6 +23,7 @@ charsets.ddl err_attr_dspace.ddl file_space.ddl + file_space_cache.ddl filter_fail.ddl non_existing.ddl infinite_loop.ddl @@ -1187,6 +1188,7 @@ ADD_H5_TEST (tboot2A 0 --enable-error-stack --boot-block tfcontents2.h5) ADD_H5_TEST (tboot2B 0 --enable-error-stack --superblock tfcontents2.h5) ADD_H5_TEST (file_space 0 --enable-error-stack -B file_space.h5) + ADD_H5_TEST (file_space_cache 0 --enable-error-stack=2 --page-buffer-size=16384 -B file_space.h5) # test -p with a non existing dataset ADD_H5ERR_MASK_TEST (tperror 1 "h5dump error: unable to get link info from \"bogus\"" --enable-error-stack -p -d bogus tfcontents1.h5) diff --git a/tools/test/h5dump/expected/file_space_cache.ddl b/tools/test/h5dump/expected/file_space_cache.ddl new file mode 100644 index 00000000000..6b8aa476a19 --- /dev/null +++ b/tools/test/h5dump/expected/file_space_cache.ddl @@ -0,0 +1,22 @@ +HDF5 "file_space.h5" { +SUPER_BLOCK { + SUPERBLOCK_VERSION 2 + FREELIST_VERSION 0 + SYMBOLTABLE_VERSION 0 + OBJECTHEADER_VERSION 0 + OFFSET_SIZE 8 + LENGTH_SIZE 8 + BTREE_RANK 16 + BTREE_LEAF 4 + ISTORE_K 32 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_NONE + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 8192 + USER_BLOCK { + USERBLOCK_SIZE 0 + } +} +GROUP "/" { +} +} diff --git a/tools/test/h5dump/expected/h5dump-help.txt b/tools/test/h5dump/expected/h5dump-help.txt index 45beb63d23b..a78d8d820ec 100644 --- a/tools/test/h5dump/expected/h5dump-help.txt +++ b/tools/test/h5dump/expected/h5dump-help.txt @@ -16,6 +16,7 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Supply S3 authentication information to "ros3" vfd. :: "(,,)" If absent or -> "(,,)", no authentication. diff --git a/tools/test/h5dump/expected/pbits/tnofilename-with-packed-bits.ddl b/tools/test/h5dump/expected/pbits/tnofilename-with-packed-bits.ddl index 45beb63d23b..a78d8d820ec 100644 --- a/tools/test/h5dump/expected/pbits/tnofilename-with-packed-bits.ddl +++ b/tools/test/h5dump/expected/pbits/tnofilename-with-packed-bits.ddl @@ -16,6 +16,7 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Supply S3 authentication information to "ros3" vfd. :: "(,,)" If absent or -> "(,,)", no authentication. diff --git a/tools/test/h5dump/expected/pbits/tpbitsIncomplete.ddl b/tools/test/h5dump/expected/pbits/tpbitsIncomplete.ddl index 45beb63d23b..a78d8d820ec 100644 --- a/tools/test/h5dump/expected/pbits/tpbitsIncomplete.ddl +++ b/tools/test/h5dump/expected/pbits/tpbitsIncomplete.ddl @@ -16,6 +16,7 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Supply S3 authentication information to "ros3" vfd. :: "(,,)" If absent or -> "(,,)", no authentication. diff --git a/tools/test/h5dump/expected/pbits/tpbitsLengthExceeded.ddl b/tools/test/h5dump/expected/pbits/tpbitsLengthExceeded.ddl index 45beb63d23b..a78d8d820ec 100644 --- a/tools/test/h5dump/expected/pbits/tpbitsLengthExceeded.ddl +++ b/tools/test/h5dump/expected/pbits/tpbitsLengthExceeded.ddl @@ -16,6 +16,7 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Supply S3 authentication information to "ros3" vfd. :: "(,,)" If absent or -> "(,,)", no authentication. diff --git a/tools/test/h5dump/expected/pbits/tpbitsLengthPositive.ddl b/tools/test/h5dump/expected/pbits/tpbitsLengthPositive.ddl index 45beb63d23b..a78d8d820ec 100644 --- a/tools/test/h5dump/expected/pbits/tpbitsLengthPositive.ddl +++ b/tools/test/h5dump/expected/pbits/tpbitsLengthPositive.ddl @@ -16,6 +16,7 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Supply S3 authentication information to "ros3" vfd. :: "(,,)" If absent or -> "(,,)", no authentication. diff --git a/tools/test/h5dump/expected/pbits/tpbitsMaxExceeded.ddl b/tools/test/h5dump/expected/pbits/tpbitsMaxExceeded.ddl index 45beb63d23b..a78d8d820ec 100644 --- a/tools/test/h5dump/expected/pbits/tpbitsMaxExceeded.ddl +++ b/tools/test/h5dump/expected/pbits/tpbitsMaxExceeded.ddl @@ -16,6 +16,7 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Supply S3 authentication information to "ros3" vfd. :: "(,,)" If absent or -> "(,,)", no authentication. diff --git a/tools/test/h5dump/expected/pbits/tpbitsOffsetExceeded.ddl b/tools/test/h5dump/expected/pbits/tpbitsOffsetExceeded.ddl index 45beb63d23b..a78d8d820ec 100644 --- a/tools/test/h5dump/expected/pbits/tpbitsOffsetExceeded.ddl +++ b/tools/test/h5dump/expected/pbits/tpbitsOffsetExceeded.ddl @@ -16,6 +16,7 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Supply S3 authentication information to "ros3" vfd. :: "(,,)" If absent or -> "(,,)", no authentication. diff --git a/tools/test/h5dump/expected/pbits/tpbitsOffsetNegative.ddl b/tools/test/h5dump/expected/pbits/tpbitsOffsetNegative.ddl index 45beb63d23b..a78d8d820ec 100644 --- a/tools/test/h5dump/expected/pbits/tpbitsOffsetNegative.ddl +++ b/tools/test/h5dump/expected/pbits/tpbitsOffsetNegative.ddl @@ -16,6 +16,7 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Supply S3 authentication information to "ros3" vfd. :: "(,,)" If absent or -> "(,,)", no authentication. diff --git a/tools/test/h5dump/testh5dump.sh.in b/tools/test/h5dump/testh5dump.sh.in index efada14934f..6a60785e959 100644 --- a/tools/test/h5dump/testh5dump.sh.in +++ b/tools/test/h5dump/testh5dump.sh.in @@ -198,6 +198,7 @@ $SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.h5.onion LIST_OTHER_TEST_FILES=" $SRC_H5DUMP_OUTFILES/charsets.ddl $SRC_H5DUMP_OUTFILES/file_space.ddl +$SRC_H5DUMP_OUTFILES/file_space_cache.ddl $SRC_H5DUMP_OUTFILES/filter_fail.ddl $SRC_H5DUMP_OUTFILES/non_existing.ddl $SRC_H5DUMP_OUTFILES/infinite_loop.ddl @@ -1254,6 +1255,7 @@ TOOLTEST tboot2.ddl --enable-error-stack -B tfcontents2.h5 TOOLTEST tboot2A.ddl --enable-error-stack --boot-block tfcontents2.h5 TOOLTEST tboot2B.ddl --enable-error-stack --superblock tfcontents2.h5 TOOLTEST file_space.ddl --enable-error-stack -B file_space.h5 +TOOLTEST file_space_cache.ddl --enable-error-stack=2 --page-buffer-size=16384 -B file_space.h5 # test -p with a non existing dataset GREPTEST ERRTXT "h5dump error: unable to get link info from" tperror.ddl --enable-error-stack -p -d bogus tfcontents1.h5 diff --git a/tools/test/h5ls/CMakeTests.cmake b/tools/test/h5ls/CMakeTests.cmake index e47c254f465..f9a489f96f0 100644 --- a/tools/test/h5ls/CMakeTests.cmake +++ b/tools/test/h5ls/CMakeTests.cmake @@ -258,6 +258,9 @@ ADD_H5_TEST (tgroup 0 -w80 tgroup.h5) ADD_H5_TEST (tgroup-3 0 -w80 tgroup.h5/g1) + # test page buffer cache command + ADD_H5_TEST (tall-pbc 0 -w80 --page-buffer-size=8192 tall.h5) + # test for displaying groups # The following combination of arguments is expected to return an error message # and return value 1 diff --git a/tools/test/h5ls/expected/help-1.ls b/tools/test/h5ls/expected/help-1.ls index 1cf733bc3a3..668f34aabe6 100644 --- a/tools/test/h5ls/expected/help-1.ls +++ b/tools/test/h5ls/expected/help-1.ls @@ -35,6 +35,7 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] -wN, --width=N Set the number of columns of output -v, --verbose Generate more verbose output -V, --version Print version number and exit + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vfd=DRIVER Use the specified virtual file driver -x, --hexdump Show raw data in hexadecimal format --s3-cred=C Supply S3 authentication information to "ros3" vfd. diff --git a/tools/test/h5ls/expected/help-2.ls b/tools/test/h5ls/expected/help-2.ls index 1cf733bc3a3..668f34aabe6 100644 --- a/tools/test/h5ls/expected/help-2.ls +++ b/tools/test/h5ls/expected/help-2.ls @@ -35,6 +35,7 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] -wN, --width=N Set the number of columns of output -v, --verbose Generate more verbose output -V, --version Print version number and exit + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vfd=DRIVER Use the specified virtual file driver -x, --hexdump Show raw data in hexadecimal format --s3-cred=C Supply S3 authentication information to "ros3" vfd. diff --git a/tools/test/h5ls/expected/help-3.ls b/tools/test/h5ls/expected/help-3.ls index 1cf733bc3a3..668f34aabe6 100644 --- a/tools/test/h5ls/expected/help-3.ls +++ b/tools/test/h5ls/expected/help-3.ls @@ -35,6 +35,7 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] -wN, --width=N Set the number of columns of output -v, --verbose Generate more verbose output -V, --version Print version number and exit + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vfd=DRIVER Use the specified virtual file driver -x, --hexdump Show raw data in hexadecimal format --s3-cred=C Supply S3 authentication information to "ros3" vfd. diff --git a/tools/test/h5ls/expected/textlinksrc-nodangle-1.ls b/tools/test/h5ls/expected/textlinksrc-nodangle-1.ls index 1cf733bc3a3..668f34aabe6 100644 --- a/tools/test/h5ls/expected/textlinksrc-nodangle-1.ls +++ b/tools/test/h5ls/expected/textlinksrc-nodangle-1.ls @@ -35,6 +35,7 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] -wN, --width=N Set the number of columns of output -v, --verbose Generate more verbose output -V, --version Print version number and exit + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vfd=DRIVER Use the specified virtual file driver -x, --hexdump Show raw data in hexadecimal format --s3-cred=C Supply S3 authentication information to "ros3" vfd. diff --git a/tools/test/h5ls/expected/tgroup-1.ls b/tools/test/h5ls/expected/tgroup-1.ls index 1cf733bc3a3..668f34aabe6 100644 --- a/tools/test/h5ls/expected/tgroup-1.ls +++ b/tools/test/h5ls/expected/tgroup-1.ls @@ -35,6 +35,7 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] -wN, --width=N Set the number of columns of output -v, --verbose Generate more verbose output -V, --version Print version number and exit + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --vfd=DRIVER Use the specified virtual file driver -x, --hexdump Show raw data in hexadecimal format --s3-cred=C Supply S3 authentication information to "ros3" vfd. diff --git a/tools/test/h5repack/expected/h5repack-help.txt b/tools/test/h5repack/expected/h5repack-help.txt index b291a6881dd..50242ef4853 100644 --- a/tools/test/h5repack/expected/h5repack-help.txt +++ b/tools/test/h5repack/expected/h5repack-help.txt @@ -10,6 +10,7 @@ usage: h5repack [OPTIONS] file1 file2 N - is an integer greater than 1, 2 displays read/write timing -V, --version Print version number and exit -n, --native Use a native HDF5 type when repacking + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --src-vol-value Value (ID) of the VOL connector to use for opening the input HDF5 file specified --src-vol-name Name of the VOL connector to use for opening the input diff --git a/tools/test/h5repack/h5repacktst.c b/tools/test/h5repack/h5repacktst.c index f3b84c4d1fd..62d3a948c92 100644 --- a/tools/test/h5repack/h5repacktst.c +++ b/tools/test/h5repack/h5repacktst.c @@ -384,7 +384,7 @@ main(void) GOERROR; if (h5repack_verify(FNAME0, FNAME0OUT, &pack_options) <= 0) GOERROR; - if (h5repack_cmp_pl(FNAME0, pack_options.fin_fapl, FNAME0OUT, pack_options.fout_fapl) <= 0) + if (h5repack_cmp_pl(FNAME0, FNAME0OUT, &pack_options) <= 0) GOERROR; if (h5repack_end(&pack_options) < 0) GOERROR; @@ -404,7 +404,7 @@ main(void) GOERROR; if (h5repack_verify(FNAME1, FNAME1OUT, &pack_options) <= 0) GOERROR; - if (h5repack_cmp_pl(FNAME1, pack_options.fin_fapl, FNAME1OUT, pack_options.fout_fapl) <= 0) + if (h5repack_cmp_pl(FNAME1, FNAME1OUT, &pack_options) <= 0) GOERROR; if (h5repack_end(&pack_options) < 0) GOERROR; @@ -424,7 +424,7 @@ main(void) GOERROR; if (h5repack_verify(FNAME2, FNAME2OUT, &pack_options) <= 0) GOERROR; - if (h5repack_cmp_pl(FNAME2, pack_options.fin_fapl, FNAME2OUT, pack_options.fout_fapl) <= 0) + if (h5repack_cmp_pl(FNAME2, FNAME2OUT, &pack_options) <= 0) GOERROR; if (h5repack_end(&pack_options) < 0) GOERROR; @@ -443,7 +443,7 @@ main(void) GOERROR; if (h5repack_verify(FNAME3, FNAME3OUT, &pack_options) <= 0) GOERROR; - if (h5repack_cmp_pl(FNAME3, pack_options.fin_fapl, FNAME3OUT, pack_options.fout_fapl) <= 0) + if (h5repack_cmp_pl(FNAME3, FNAME3OUT, &pack_options) <= 0) GOERROR; if (h5repack_end(&pack_options) < 0) GOERROR; @@ -1127,7 +1127,7 @@ main(void) GOERROR; if (h5repack_verify(FNAME7, FNAME7OUT, &pack_options) <= 0) GOERROR; - if (h5repack_cmp_pl(FNAME7, pack_options.fin_fapl, FNAME7OUT, pack_options.fout_fapl) <= 0) + if (h5repack_cmp_pl(FNAME7, FNAME7OUT, &pack_options) <= 0) GOERROR; if (h5repack_end(&pack_options) < 0) GOERROR; diff --git a/tools/test/h5stat/CMakeTests.cmake b/tools/test/h5stat/CMakeTests.cmake index f6287406ee8..943551b4bec 100644 --- a/tools/test/h5stat/CMakeTests.cmake +++ b/tools/test/h5stat/CMakeTests.cmake @@ -28,6 +28,7 @@ h5stat_notexist h5stat_nofile h5stat_filters + h5stat_filters-cache h5stat_filters-file h5stat_filters-F h5stat_filters-d @@ -192,6 +193,8 @@ ADD_H5_TEST (h5stat_filters-dT 0 -dT h5stat_filters.h5) ADD_H5_TEST (h5stat_filters-UD 0 -D h5stat_filters.h5) ADD_H5_TEST (h5stat_filters-UT 0 -T h5stat_filters.h5) +# Test for page buffer cache size option + ADD_H5_TEST (h5stat_filters-cache 0 --enable-error-stack=2 --page-buffer-size=8192 h5stat_filters.h5) # h5stat_tsohm.h5 is a copy of ../../../test/tsohm.h5 generated by tsohm.c # as of release 1.8.7-snap0 (on a 64-bit machine) ADD_H5_TEST (h5stat_tsohm 0 h5stat_tsohm.h5) diff --git a/tools/test/h5stat/expected/h5stat_filters-cache.ddl b/tools/test/h5stat/expected/h5stat_filters-cache.ddl new file mode 100644 index 00000000000..7383f0b31ff --- /dev/null +++ b/tools/test/h5stat/expected/h5stat_filters-cache.ddl @@ -0,0 +1,99 @@ +Filename: h5stat_filters.h5 +File information + # of unique groups: 1 + # of unique datasets: 15 + # of unique named datatypes: 1 + # of unique links: 0 + # of unique other: 0 + Max. # of links to object: 1 + Max. # of objects in group: 16 +File space information for file metadata (in bytes): + Superblock: 96 + Superblock extension: 0 + User block: 0 + Object headers: (total/unused) + Groups: 40/0 + Datasets(exclude compact data): 4128/1088 + Datatypes: 80/0 + Groups: + B-tree/List: 1200 + Heap: 384 + Attributes: + B-tree/List: 0 + Heap: 0 + Chunked datasets: + Index: 31392 + Datasets: + Heap: 72 + Shared Messages: + Header: 0 + B-tree/List: 0 + Heap: 0 + Free-space managers: + Header: 0 + Amount of free space: 0 +Small groups (with 0 to 9 links): + Total # of small groups: 0 +Group bins: + # of groups with 10 - 99 links: 1 + Total # of groups: 1 +Dataset dimension information: + Max. rank of datasets: 2 + Dataset ranks: + # of dataset with rank 1: 1 + # of dataset with rank 2: 14 +1-D Dataset information: + Max. dimension size of 1-D datasets: 100 + Small 1-D datasets (with dimension sizes 0 to 9): + Total # of small datasets: 0 + 1-D Dataset dimension bins: + # of datasets with dimension size 100 - 999: 1 + Total # of datasets: 1 +Dataset storage information: + Total raw data size: 9046 + Total external raw data size: 400 +Dataset layout information: + Dataset layout counts[COMPACT]: 1 + Dataset layout counts[CONTIG]: 2 + Dataset layout counts[CHUNKED]: 12 + Dataset layout counts[VIRTUAL]: 0 + Number of external files : 2 +Dataset filters information: + Number of datasets with: + NO filter: 7 + GZIP filter: 2 + SHUFFLE filter: 2 + FLETCHER32 filter: 2 + SZIP filter: 2 + NBIT filter: 2 + SCALEOFFSET filter: 1 + USER-DEFINED filter: 1 +Dataset datatype information: + # of unique datatypes used by datasets: 2 + Dataset datatype #0: + Count (total/named) = (14/0) + Size (desc./elmt) = (14/4) + Dataset datatype #1: + Count (total/named) = (1/0) + Size (desc./elmt) = (14/4) + Total dataset datatype count: 15 +Small # of attributes (objects with 1 to 10 attributes): + Total # of objects with small # of attributes: 0 +Attribute bins: + Total # of objects with attributes: 0 + Max. # of attributes to objects: 0 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes +Summary of file space information: + File metadata: 37392 bytes + Raw data: 9046 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 258 bytes +Total space: 46696 bytes +External raw data: 400 bytes diff --git a/tools/test/h5stat/expected/h5stat_help1.ddl b/tools/test/h5stat/expected/h5stat_help1.ddl index 54d6a318eda..82f1cfff3ee 100644 --- a/tools/test/h5stat/expected/h5stat_help1.ddl +++ b/tools/test/h5stat/expected/h5stat_help1.ddl @@ -25,6 +25,7 @@ usage: h5stat [OPTIONS] file than 0. The default threshold is 10. -s, --freespace Print free space information -S, --summary Print summary of file space information + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Access file on S3, using provided credential :: (region,id,key) If == "(,,)", no authentication is used. diff --git a/tools/test/h5stat/expected/h5stat_help2.ddl b/tools/test/h5stat/expected/h5stat_help2.ddl index 54d6a318eda..82f1cfff3ee 100644 --- a/tools/test/h5stat/expected/h5stat_help2.ddl +++ b/tools/test/h5stat/expected/h5stat_help2.ddl @@ -25,6 +25,7 @@ usage: h5stat [OPTIONS] file than 0. The default threshold is 10. -s, --freespace Print free space information -S, --summary Print summary of file space information + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Access file on S3, using provided credential :: (region,id,key) If == "(,,)", no authentication is used. diff --git a/tools/test/h5stat/expected/h5stat_nofile.ddl b/tools/test/h5stat/expected/h5stat_nofile.ddl index 54d6a318eda..82f1cfff3ee 100644 --- a/tools/test/h5stat/expected/h5stat_nofile.ddl +++ b/tools/test/h5stat/expected/h5stat_nofile.ddl @@ -25,6 +25,7 @@ usage: h5stat [OPTIONS] file than 0. The default threshold is 10. -s, --freespace Print free space information -S, --summary Print summary of file space information + --page-buffer-size=N Set the page buffer cache size, N=non-negative integers --s3-cred= Access file on S3, using provided credential :: (region,id,key) If == "(,,)", no authentication is used. diff --git a/tools/test/h5stat/testh5stat.sh.in b/tools/test/h5stat/testh5stat.sh.in index bc48494f59b..b44c43921de 100644 --- a/tools/test/h5stat/testh5stat.sh.in +++ b/tools/test/h5stat/testh5stat.sh.in @@ -84,6 +84,7 @@ $SRC_H5STAT_OUTFILES/h5stat_help2.ddl $SRC_H5STAT_OUTFILES/h5stat_notexist.ddl $SRC_H5STAT_OUTFILES/h5stat_nofile.ddl $SRC_H5STAT_OUTFILES/h5stat_filters.ddl +$SRC_H5STAT_OUTFILES/h5stat_filters-cache.ddl $SRC_H5STAT_OUTFILES/h5stat_filters-file.ddl $SRC_H5STAT_OUTFILES/h5stat_filters-F.ddl $SRC_H5STAT_OUTFILES/h5stat_filters-d.ddl @@ -290,6 +291,8 @@ TOOLTEST h5stat_filters-g.ddl -g h5stat_filters.h5 TOOLTEST h5stat_filters-dT.ddl -dT h5stat_filters.h5 TOOLTEST h5stat_filters-UD.ddl -D h5stat_filters.h5 TOOLTEST h5stat_filters-UT.ddl -T h5stat_filters.h5 +# Test for page buffer cache size option +TOOLTEST h5stat_filters-cache.ddl --page-buffer-size=8192 h5stat_filters.h5 # # h5stat_tsohm.h5 is a copy of ../../../test/tsohm.h5 generated by tsohm.c # as of release 1.8.7-snap0 (on a 64-bit machine)