From 83c7f928f97b56a7a83f7f81b207cbe082136b1f Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Tue, 20 Dec 2022 16:49:33 -0800 Subject: [PATCH] Brings the following changesets over from develop: (#2328) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit b9244a85d9f1cc5e9bbec61ca73c0cbd9c4cf249 Align arg types of H5D_chunk_iter_op_t with H5Dget_chunk_info (#2074) 70cf2c390bc2eef8e57f8fa023341011e2d02d9d Removed idioms and misc. text clean-up (#2320) 8102fa8c972bdc0d8fd8f3dae604e070893150d6 Only document Fortran functions (#2319) 784061b15e176b9919c19a220ce278a9f4cddf0e moved onion VFD to FAPL group (#2321) 6b6bcdead66f0456ac0528683faac6a8e48b6565 Hdffv 11052 (#2315) 10c693a04ff0c4a5219879d7f8900157dcbece66 Update hdf5_header.html 0cb58080875070db09b5ecae92482519d58872bc Hdffv 11052 (#2303) a1c81eda203addced514ef655f7a9079f3f0bb04 added doc. warning for H5Literate_async return value (#2295) 502b32b0f22a4bcf6333c85c256db34162c2764a Updated H5ES documenation (#2293) a9036005c3916e6fda0296026323f00d043300f8 Fix for HDFFV-11052: h5debug fails on a corrupted file (h5_nrefs_POC)… (#2291) --- doxygen/Doxyfile.in | 10 +++++- doxygen/dox/ReferenceManual.dox | 13 +++----- doxygen/hdf5_header.html | 2 +- release_docs/RELEASE.txt | 19 ++++++++++++ src/H5D.c | 16 +++++----- src/H5Dchunk.c | 4 +-- src/H5Dpublic.h | 2 +- src/H5ESpublic.h | 10 +++--- src/H5FDonion.h | 8 ++--- src/H5Fint.c | 4 ++- src/H5Lpublic.h | 5 +++ src/H5VLnative_file.c | 52 ++++++++++++++++++-------------- test/CMakeTests.cmake | 1 + test/chunk_info.c | 18 +++++------ test/cve_2020_10812.h5 | Bin 0 -> 2565 bytes test/tmisc.c | 47 +++++++++++++++++++++++++++++ tools/src/misc/h5debug.c | 8 +++-- 17 files changed, 153 insertions(+), 66 deletions(-) create mode 100755 test/cve_2020_10812.h5 diff --git a/doxygen/Doxyfile.in b/doxygen/Doxyfile.in index f45f7461592..28c73272dda 100644 --- a/doxygen/Doxyfile.in +++ b/doxygen/Doxyfile.in @@ -897,7 +897,15 @@ EXCLUDE_SYMLINKS = NO # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* -EXCLUDE_PATTERNS = +EXCLUDE_PATTERNS = */fortran/test/* +EXCLUDE_PATTERNS += */fortran/testpar/* +EXCLUDE_PATTERNS += */fortran/examples/* +EXCLUDE_PATTERNS += */fortran/src/*.c +EXCLUDE_PATTERNS += */fortran/src/*.h +EXCLUDE_PATTERNS += */hl/fortran/examples/* +EXCLUDE_PATTERNS += */hl/fortran/test/* +EXCLUDE_PATTERNS += */hl/fortran/src/*.c +EXCLUDE_PATTERNS += */hl/fortran/src/*.h # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the diff --git a/doxygen/dox/ReferenceManual.dox b/doxygen/dox/ReferenceManual.dox index e79de679f10..b9bcd498357 100644 --- a/doxygen/dox/ReferenceManual.dox +++ b/doxygen/dox/ReferenceManual.dox @@ -145,23 +145,19 @@ Functions with \ref ASYNC
- -Mind the gap - Follow these simple rules and stay out of trouble: -\li \Bold{Handle discipline:} The HDF5 C-API is rife with handles or +\li \Bold{Handle discipline:} The HDF5 API is rife with handles or identifiers, which you typically obtain by creating new HDF5 items, copying - items, or retrieving facets of items. \Emph{You acquire a handle, you own it!} - (Colin Powell) In other words, you are responsible for releasing the underlying + items, or retrieving facets of items. Consequently, \Bold{and most importantly}, you are + responsible for releasing the underlying resources via the matching \Code{H5*close()} call, or deal with the consequences of resource leakage. \li \Bold{Closed means closed:} Do not pass identifiers that were previously \Code{H5*close()}-d to other API functions! It will generate an error. \li \Bold{Dynamic memory allocation:} The API contains a few functions in which the HDF5 library dynamically allocates memory on the caller's behalf. The caller owns - this memory and eventually must free it by calling H5free_memory(). (\Bold{Not} - the `free` function \Emph{du jour}!) + this memory and eventually must free it by calling H5free_memory() and not language-explicit memory functions. \li \Bold{Be careful with that saw:} Do not modify the underlying collection when an iteration is in progress! \li \Bold{Use of locations:} Certain API functions, typically called \Code{H5***_by_name} @@ -169,7 +165,6 @@ Follow these simple rules and stay out of trouble: If the identifier fully specifies the object in question, pass \Code{'.'} (a dot) for the name! -Break a leg! diff --git a/doxygen/hdf5_header.html b/doxygen/hdf5_header.html index 23f41f9b501..36a32653ab9 100644 --- a/doxygen/hdf5_header.html +++ b/doxygen/hdf5_header.html @@ -21,7 +21,7 @@ -
Please, help us to better know about our user community by answering the following short survey: https://www.hdfgroup.org/website-survey/
+
Please, help us to better serve our user community by answering the following short survey: https://www.hdfgroup.org/website-survey/
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 930bc421a42..1bccc70d679 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -171,6 +171,25 @@ Bug Fixes since HDF5-1.13.3 release =================================== Library ------- + - Seg fault on file close + + h5debug fails at file close with core dump on a file that has an + illegal file size in its cache image. In H5F_dest(), the library + performs all the closing operations for the file and keeps track of + the error encountered when reading the file cache image. + At the end of the routine, it frees the file's file structure and + returns error. Due to the error return, the file object is not removed + from the ID node table. This eventually causes assertion failure in + H5VL__native_file_close() when the library finally exits and tries to + access that file object in the table for closing. + + The closing routine, H5F_dest(), will not free the file structure if + there is error, keeping a valid file structure in the ID node table. + It will be freed later in H5VL__native_file_close() when the + library exits and terminates the file package. + + (VC - 2022/12/14, HDFFV-11052, CVE-2020-10812) + - Fix CVE-2018-13867 / GHSA-j8jr-chrh-qfrf Validate location (offset) of the accumulated metadata when comparing. diff --git a/src/H5D.c b/src/H5D.c index d9213566cb0..006f4a9d88f 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -2515,18 +2515,18 @@ H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filte * * typedef int (*H5D_chunk_iter_op_t)( * const hsize_t *offset, - * uint32_t filter_mask, + * unsigned filter_mask, * haddr_t addr, - * uint32_t nbytes, + * hsize_t size, * void *op_data); * * H5D_chunk_iter_op_t parameters: - * hsize_t *offset; IN/OUT: Array of starting logical coordinates of chunk. - * uint32_t filter_mask; IN: Filter mask of chunk. - * haddr_t addr; IN: Offset in file of chunk data. - * uint32_t nbytes; IN: Size in number of bytes of chunk data in file. - * void *op_data; IN/OUT: Pointer to any user-defined data - * associated with the operation. + * hsize_t *offset; IN/OUT: Logical position of the chunk’s first element in units of dataset + * elements + * unsigned filter_mask; IN: Bitmask indicating the filters used when the chunk was written haddr_t + * addr; IN: Chunk address in the file + * hsize_t; IN: Chunk size in bytes, 0 if the chunk does not exist + * void *op_data; IN/OUT: Pointer to any user-defined data associated with the operation. * * The return values from an operator are: * Zero (H5_ITER_CONT) causes the iterator to continue, returning zero when all diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 40f8359f2cf..830560d454a 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -8278,8 +8278,8 @@ H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata) FUNC_ENTER_PACKAGE_NOERR /* Check for callback failure and pass along return value */ - if ((ret_value = (data->op)(offset, chunk_rec->filter_mask, chunk_rec->chunk_addr, chunk_rec->nbytes, - data->op_data)) < 0) + if ((ret_value = (data->op)(offset, (unsigned)chunk_rec->filter_mask, chunk_rec->chunk_addr, + (hsize_t)chunk_rec->nbytes, data->op_data)) < 0) HERROR(H5E_DATASET, H5E_CANTNEXT, "iteration operator failed"); FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index ac76bc8a8c8..f7d208dd9f0 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -238,7 +238,7 @@ typedef herr_t (*H5D_gather_func_t)(const void *dst_buf, size_t dst_buf_bytes_us * \li A negative (#H5_ITER_ERROR) causes the iterator to immediately * return that value, indicating failure. */ -typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t size, +typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *op_data); //! diff --git a/src/H5ESpublic.h b/src/H5ESpublic.h index 3d46e827539..ecfd08f7a27 100644 --- a/src/H5ESpublic.h +++ b/src/H5ESpublic.h @@ -189,7 +189,7 @@ H5_DLL herr_t H5ESwait(hid_t es_id, uint64_t timeout, size_t *num_in_progress, h * \param[out] err_occurred Status indicating if error is present in the event set * \returns \herr_t * - * \details H5ESget_count() attempts to cancel operations in an event set specified + * \details H5EScancel() attempts to cancel operations in an event set specified * by \p es_id. H5ES_NONE is a valid value for \p es_id, but functions as a no-op. * * \since 1.13.0 @@ -217,14 +217,14 @@ H5_DLL herr_t H5ESget_count(hid_t es_id, size_t *count); /** * \ingroup H5ES * - * \brief Retrieves the next operation counter to be assigned in an event set + * \brief Retrieves the accumulative operation counter for an event set * * \es_id - * \param[out] counter The next counter value to be assigned to an event + * \param[out] counter The accumulative counter value for an event set * \returns \herr_t * - * \details H5ESget_op_counter() retrieves the \p counter that will be assigned - * to the next operation inserted into the event set \p es_id. + * \details H5ESget_op_counter() retrieves the current accumulative count of + * event set operations since the event set creation of \p es_id. * * \note This is designed for wrapper libraries mainly, to use as a mechanism * for matching operations inserted into the event set with possible diff --git a/src/H5FDonion.h b/src/H5FDonion.h index 63c2d778a35..0e605d0fd80 100644 --- a/src/H5FDonion.h +++ b/src/H5FDonion.h @@ -142,11 +142,11 @@ H5_DLL hid_t H5FD_onion_init(void); /** * -------------------------------------------------------------------------- - * \ingroup H5P + * \ingroup FAPL * * \brief get the onion info from the file access property list * - * \param[in] fapl_id The ID of the file access property list + * \fapl_id * \param[out] fa_out The pointer to the structure H5FD_onion_fapl_info_t * * \return \herr_t @@ -159,11 +159,11 @@ H5_DLL herr_t H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out); /** * -------------------------------------------------------------------------- - * \ingroup H5P + * \ingroup FAPL * * \brief set the onion info for the file access property list * - * \param[in] fapl_id The ID of the file access property list + * \fapl_id * \param[in] fa The pointer to the structure H5FD_onion_fapl_info_t * * \return \herr_t diff --git a/src/H5Fint.c b/src/H5Fint.c index 2c1b4b2c3c0..7ad35fc552d 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -1615,7 +1615,9 @@ H5F__dest(H5F_t *f, hbool_t flush) if (H5FO_top_dest(f) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "problems closing file") f->shared = NULL; - f = H5FL_FREE(H5F_t, f); + + if (ret_value >= 0) + f = H5FL_FREE(H5F_t, f); FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__dest() */ diff --git a/src/H5Lpublic.h b/src/H5Lpublic.h index 6f6c6382123..60e83f2f200 100644 --- a/src/H5Lpublic.h +++ b/src/H5Lpublic.h @@ -923,6 +923,11 @@ H5_DLL herr_t H5Literate2(hid_t grp_id, H5_index_t idx_type, H5_iter_order_t ord /** * -------------------------------------------------------------------------- * \ingroup ASYNC + * + * \warning The returned value of the callback routine op will not be set + * in the return value for H5Literate_async(), so the \p herr_t value + * should not be used for determining the return state of the callback routine. + * * \async_variant_of{H5Literate} */ #ifndef H5_DOXYGEN diff --git a/src/H5VLnative_file.c b/src/H5VLnative_file.c index 907a12dedce..f2f0ea7d19e 100644 --- a/src/H5VLnative_file.c +++ b/src/H5VLnative_file.c @@ -753,29 +753,35 @@ H5VL__native_file_close(void *file, hid_t H5_ATTR_UNUSED dxpl_id, void H5_ATTR_U FUNC_ENTER_PACKAGE /* This routine should only be called when a file ID's ref count drops to zero */ - HDassert(H5F_ID_EXISTS(f)); - - /* Flush file if this is the last reference to this id and we have write - * intent, unless it will be flushed by the "shared" file being closed. - * This is only necessary to replicate previous behaviour, and could be - * disabled by an option/property to improve performance. - */ - if ((H5F_NREFS(f) > 1) && (H5F_INTENT(f) & H5F_ACC_RDWR)) { - /* Get the file ID corresponding to the H5F_t struct */ - if (H5I_find_id(f, H5I_FILE, &file_id) < 0 || H5I_INVALID_HID == file_id) - HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "invalid ID") - - /* Get the number of references outstanding for this file ID */ - if ((nref = H5I_get_ref(file_id, FALSE)) < 0) - HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "can't get ID ref count") - if (nref == 1) - if (H5F__flush(f) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cache") - } /* end if */ - - /* Close the file */ - if (H5F__close(f) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "can't close file") + HDassert(f->shared == NULL || H5F_ID_EXISTS(f)); + + if (f->shared == NULL) + f = H5FL_FREE(H5F_t, f); + + else { + + /* Flush file if this is the last reference to this id and we have write + * intent, unless it will be flushed by the "shared" file being closed. + * This is only necessary to replicate previous behaviour, and could be + * disabled by an option/property to improve performance. + */ + if ((H5F_NREFS(f) > 1) && (H5F_INTENT(f) & H5F_ACC_RDWR)) { + /* Get the file ID corresponding to the H5F_t struct */ + if (H5I_find_id(f, H5I_FILE, &file_id) < 0 || H5I_INVALID_HID == file_id) + HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "invalid ID") + + /* Get the number of references outstanding for this file ID */ + if ((nref = H5I_get_ref(file_id, FALSE)) < 0) + HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "can't get ID ref count") + if (nref == 1) + if (H5F__flush(f) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cache") + } /* end if */ + + /* Close the file */ + if (H5F__close(f) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "can't close file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index 74f63f43aed..508619430cc 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -126,6 +126,7 @@ set (HDF5_REFERENCE_TEST_FILES btree_idx_1_6.h5 btree_idx_1_8.h5 corrupt_stab_msg.h5 + cve_2020_10812.h5 deflate.h5 family_v16-000000.h5 family_v16-000001.h5 diff --git a/test/chunk_info.c b/test/chunk_info.c index e38752f7eb0..5651b26cb6c 100644 --- a/test/chunk_info.c +++ b/test/chunk_info.c @@ -1508,9 +1508,9 @@ test_chunk_info_version2_btrees(const char *filename, hid_t fapl) typedef struct chunk_iter_info_t { hsize_t offset[2]; - uint32_t filter_mask; + unsigned filter_mask; haddr_t addr; - uint32_t nbytes; + hsize_t size; } chunk_iter_info_t; typedef struct chunk_iter_udata_t { @@ -1519,7 +1519,7 @@ typedef struct chunk_iter_udata_t { } chunk_iter_udata_t; static int -iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbytes, void *op_data) +iter_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *op_data) { chunk_iter_udata_t *cidata = (chunk_iter_udata_t *)op_data; int idx = cidata->last_index + 1; @@ -1528,7 +1528,7 @@ iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbyt cidata->chunk_info[idx].offset[1] = offset[1]; cidata->chunk_info[idx].filter_mask = filter_mask; cidata->chunk_info[idx].addr = addr; - cidata->chunk_info[idx].nbytes = nbytes; + cidata->chunk_info[idx].size = size; cidata->last_index++; @@ -1536,8 +1536,8 @@ iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbyt } static int -iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filter_mask, - haddr_t H5_ATTR_UNUSED addr, uint32_t H5_ATTR_UNUSED nbytes, void *op_data) +iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, unsigned H5_ATTR_UNUSED filter_mask, + haddr_t H5_ATTR_UNUSED addr, hsize_t H5_ATTR_UNUSED size, void *op_data) { chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; *chunk_info += 1; @@ -1545,8 +1545,8 @@ iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filte } static int -iter_cb_fail(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filter_mask, - haddr_t H5_ATTR_UNUSED addr, uint32_t H5_ATTR_UNUSED nbytes, void *op_data) +iter_cb_fail(const hsize_t H5_ATTR_UNUSED *offset, unsigned H5_ATTR_UNUSED filter_mask, + haddr_t H5_ATTR_UNUSED addr, hsize_t H5_ATTR_UNUSED size, void *op_data) { chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; *chunk_info += 1; @@ -1718,7 +1718,7 @@ test_basic_query(hid_t fapl) FAIL_PUTS_ERROR("offset[1] mismatch"); if (chunk_infos[0].filter_mask != 0) FAIL_PUTS_ERROR("filter mask mismatch"); - if (chunk_infos[0].nbytes != 96) + if (chunk_infos[0].size != 96) FAIL_PUTS_ERROR("size mismatch"); if (chunk_infos[1].offset[0] != CHUNK_NX) diff --git a/test/cve_2020_10812.h5 b/test/cve_2020_10812.h5 new file mode 100755 index 0000000000000000000000000000000000000000..a20369da0db50a0d12400e9f886f2a431fbdfe6e GIT binary patch literal 2565 zcmeD5aB<`1lHy|G;9!6O11N))3&J=7<-f7G)6K{Lf(#5DP%#OH1_l!_n-L@o1_BU@ zi(vvMgaxDj(+B`_7|ufp*`h{i7i;8UmvsK 0) H5Pclose(fapl); - if (fid > 0) - H5Fclose(fid); + if (fid > 0) { + if (H5Fclose(fid) < 0) { + HDfprintf(stderr, "Error in closing file!\n"); + exit_value = 1; + } + } /* Pop API context */ if (api_ctx_pushed)