Merge remote-tracking branch 'origin/develop' into vol_cleanup

config/cmake/examples/HDF5_Examples.cmake.in

@@ -42,6 +42,14 @@ if(DEFINED CTEST_SCRIPT_ARG)
   endforeach()
 endif()
 
+if(${CTEST_VSVERS} STREQUAL "64_VS2022") # 64-bit Visual Studio 2022
+  set(CTEST_CMAKE_GENERATOR "Visual Studio 17 2022")
+  set(CMAKE_GENERATOR_ARCHITECTURE "x64")
+elseif(${VS_VERS} STREQUAL "64_VS2019") # 64-bit Visual Studio 2019
+  set(CTEST_CMAKE_GENERATOR "Visual Studio 16 2019")
+  set(CMAKE_GENERATOR_ARCHITECTURE "x64")
+endif()
+
 ###################################################################
 ### Following Line is one of [Release, RelWithDebInfo, Debug] #####
 set(CTEST_CONFIGURATION_TYPE "$ENV{CMAKE_CONFIG_TYPE}")

config/cmake/grepTest.cmake

@@ -116,7 +116,7 @@ if (TEST_ERRREF)
               RESULT_VARIABLE TEST_ERRREF_RESULT
           )
         endif ()
-        message (FATAL_ERROR "Failed: The error output of ${TEST_PROGRAM} did not contain ${TEST_ERRREF}")
+        message (FATAL_ERROR "Failed: The error output of ${TEST_PROGRAM} did not contain '${TEST_ERRREF}'. Error output was: '${TEST_ERR_STREAM}'")
       endif ()
     endif ()
   endif ()

release_docs/RELEASE.txt

@@ -840,6 +840,19 @@ Bug Fixes since HDF5-1.14.0 release
       and H5VLget_connector_id_by_value should be closed to avoid resource
       leaks.
 
+    - Fixed a bug with large external datasets
+
+      When performing a large I/O on an external dataset, the library would only
+      issue a single read or write system call. This could cause errors or cause
+      the data to be incorrect. These calls do not guarantee that they will
+      process the entire I/O request, and may need to be called multiple times
+      to complete the I/O, advancing the buffer and reducing the size by the
+      amount actually processed by read or write each time. Implemented this
+      algorithm for external datasets in both the read and write cases.
+
+      Fixes GitHub #4216
+      Fixes h5py GitHub #2394
+
     - Fixed a bug in the Subfiling VFD that could cause a buffer over-read
       and memory allocation failures
 

release_docs/USING_CMake_Examples.txt

@@ -56,6 +56,11 @@ Default installation process:
        The default ctest configuration is defined as "Release". It can be changed
            with the CTEST_CONFIGURATION_TYPE script option. Note that this must
            be the same as the value used with the -C command line option.
+           On Windows, you can set the CTEST_VSVERS script option to either
+               64_VS2022 or 64_VS2019. Alternately, you can set the script
+               CTEST_CMAKE_GENERATOR option to "Visual Studio 16 2019" or "Visual Studio 17 2022",
+               and the CMAKE_GENERATOR_ARCHITECTURE script option to "x64".
+
        The default build configuration is defined to build and use static libraries.
 
        Shared libraries and other options can be changed by editing the
@@ -69,15 +74,15 @@ Default installation process:
     When executed, the ctest script will save the results to the log file, test.log, as
     indicated by the ctest command. If you wish to see more build and test information,
     add "-VV" to the ctest command. The output should show;
-          100% tests passed, 0 tests failed out of 156.
+          100% tests passed, 0 tests failed out of 206.
 
 
 ========================================================================
 III. Defaults in the HDF5_Examples_options.cmake file
 ========================================================================
 
-####  DEFAULT:                                                                            ###
-####         BUILD_SHARED_LIBS:BOOL=OFF                                                   ###
+####  DEFAULT:                                                                           ###
+####         BUILD_SHARED_LIBS:BOOL=OFF                                                  ###
 ####         H5EX_BUILD_C:BOOL=ON                                                        ###
 ####         H5EX_BUILD_CXX:BOOL=OFF                                                     ###
 ####         H5EX_BUILD_FORTRAN:BOOL=OFF                                                 ###

src/H5Defl.c

@@ -278,12 +278,12 @@ H5D__efl_read(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t size
 {
     int    fd = -1;
     size_t to_read;
+    size_t left_to_read;
 #ifndef NDEBUG
     hsize_t tempto_read;
 #endif /* NDEBUG */
     hsize_t skip = 0;
     haddr_t cur;
-    ssize_t n;
     size_t  u;                   /* Local index variable */
     char   *full_name = NULL;    /* File name with prefix */
     herr_t  ret_value = SUCCEED; /* Return value */
@@ -325,15 +325,43 @@ H5D__efl_read(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t size
 #else  /* NDEBUG */
         to_read  = MIN((size_t)(efl->slot[u].size - skip), (hsize_t)size);
 #endif /* NDEBUG */
-        if ((n = HDread(fd, buf, to_read)) < 0)
-            HGOTO_ERROR(H5E_EFL, H5E_READERROR, FAIL, "read error in external raw data file");
-        else if ((size_t)n < to_read)
-            memset(buf + n, 0, to_read - (size_t)n);
+
+        /* Inner loop - read to_read bytes from a single external file */
+        left_to_read = to_read;
+        while (left_to_read > 0) {
+            h5_posix_io_t     bytes_in   = 0;  /* # of bytes to read       */
+            h5_posix_io_ret_t bytes_read = -1; /* # of bytes actually read */
+
+            /* Trying to read more bytes than the return type can handle is
+             * undefined behavior in POSIX.
+             */
+            if (left_to_read > H5_POSIX_MAX_IO_BYTES)
+                bytes_in = H5_POSIX_MAX_IO_BYTES;
+            else
+                bytes_in = (h5_posix_io_t)left_to_read;
+
+            do {
+                bytes_read = HDread(fd, buf, bytes_in);
+            } while (-1 == bytes_read && EINTR == errno);
+
+            if (bytes_read < 0)
+                HGOTO_ERROR(H5E_EFL, H5E_READERROR, FAIL, "read error in external raw data file");
+
+            if (0 == bytes_read) {
+                /* End of file on disk, fill the remaining sectors to be read from this file with 0 */
+                memset(buf, 0, left_to_read);
+                bytes_read = (h5_posix_io_ret_t)left_to_read;
+            } /* end if */
+
+            left_to_read -= (size_t)bytes_read;
+            buf += bytes_read;
+        }
+
+        /* Prepare to advance to next external file */
         full_name = (char *)H5MM_xfree(full_name);
         HDclose(fd);
         fd = -1;
         size -= to_read;
-        buf += to_read;
         skip = 0;
         u++;
     } /* end while */
@@ -364,6 +392,7 @@ H5D__efl_write(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t siz
 {
     int    fd = -1;
     size_t to_write;
+    size_t left_to_write;
 #ifndef NDEBUG
     hsize_t tempto_write;
 #endif /* NDEBUG */
@@ -414,13 +443,39 @@ H5D__efl_write(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t siz
 #else  /* NDEBUG */
         to_write = MIN((size_t)(efl->slot[u].size - skip), size);
 #endif /* NDEBUG */
-        if ((size_t)HDwrite(fd, buf, to_write) != to_write)
-            HGOTO_ERROR(H5E_EFL, H5E_READERROR, FAIL, "write error in external raw data file");
+
+        /* Inner loop - write to_write bytes to a single external file */
+        left_to_write = to_write;
+        while (left_to_write > 0) {
+            h5_posix_io_t     bytes_in    = 0;  /* # of bytes to write         */
+            h5_posix_io_ret_t bytes_wrote = -1; /* # of bytes actually written */
+
+            /* Trying to write more bytes than the return type can handle is
+             * undefined behavior in POSIX.
+             */
+            if (left_to_write > H5_POSIX_MAX_IO_BYTES)
+                bytes_in = H5_POSIX_MAX_IO_BYTES;
+            else
+                bytes_in = (h5_posix_io_t)left_to_write;
+
+            do {
+                bytes_wrote = HDwrite(fd, buf, bytes_in);
+            } while (-1 == bytes_wrote && EINTR == errno);
+
+            if (bytes_wrote < 0)
+                HGOTO_ERROR(H5E_EFL, H5E_WRITEERROR, FAIL, "write error in external raw data file");
+            if (bytes_wrote == 0)
+                HGOTO_ERROR(H5E_EFL, H5E_WRITEERROR, FAIL, "wrote 0 bytes to external raw data file");
+
+            left_to_write -= (size_t)bytes_wrote;
+            buf += bytes_wrote;
+        }
+
+        /* Prepare to advance to next external file */
         full_name = (char *)H5MM_xfree(full_name);
         HDclose(fd);
         fd = -1;
         size -= to_write;
-        buf += to_write;
         skip = 0;
         u++;
     } /* end while */

src/H5TS.c

@@ -67,10 +67,12 @@ H5TS_api_info_t H5TS_api_info_p;
 /*--------------------------------------------------------------------------
  * Function:    H5TSmutex_acquire
  *
- * Purpose:     Attempts to acquire the HDF5 library global lock
+ * Purpose:     Attempts to acquire the HDF5 library global lock. Should be preceded by a call to
+ *              H5TSmutex_release().
  *
- * Note:        On success, the 'acquired' flag indicates if the HDF5 library
- *              global lock was acquired.
+ * Parameters:
+ *              lock_count; IN: The lock count that was held on the mutex before its release
+ *              acquired; OUT: Whether the HDF5 library global lock was acquired
  *
  * Return:      Non-negative on success / Negative on failure
  *
@@ -118,10 +120,17 @@ H5TSmutex_get_attempt_count(unsigned *count)
 /*--------------------------------------------------------------------------
  * Function:    H5TSmutex_release
  *
- * Purpose:     Releases the HDF5 library global lock
+ * Purpose:     Releases the HDF5 library global lock. Should be followed by a call to H5TSmutex_acquire().
  *
- * Return:      Non-negative on success / Negative on failure
+ *              This should be used by applications to temporarily release the lock in order to either perform
+ *              multi-threaded work of their own or yield control to another thread using HDF5. The value
+ *              returned in lock_count should be provided to H5TSmutex_acquire() in order to resume a
+ *              consistent library state.
+ *
+ * Parameters:
+ *              lock_count; OUT: The current lock count for the calling thread.
  *
+ * Return:      Non-negative on success / Negative on failure
  *--------------------------------------------------------------------------
  */
 herr_t

test/ttsafe_acreate.c

@@ -63,7 +63,11 @@ tts_acreate(void)
     int    buffer, i;
     herr_t status;
 
-    ttsafe_name_data_t *attrib_data;
+    ttsafe_name_data_t *attrib_data[NUM_THREADS];
+
+    char *attribute_name = NULL;
+
+    memset(attrib_data, 0, sizeof(attrib_data));
 
     /*
      * Create an HDF5 file using H5F_ACC_TRUNC access, default file
@@ -97,12 +101,12 @@ tts_acreate(void)
      * with the dataset
      */
     for (i = 0; i < NUM_THREADS; i++) {
-        attrib_data                = (ttsafe_name_data_t *)malloc(sizeof(ttsafe_name_data_t));
-        attrib_data->dataset       = dataset;
-        attrib_data->datatype      = datatype;
-        attrib_data->dataspace     = dataspace;
-        attrib_data->current_index = i;
-        if (H5TS_thread_create(&threads[i], tts_acreate_thread, attrib_data) < 0)
+        attrib_data[i]                = (ttsafe_name_data_t *)malloc(sizeof(ttsafe_name_data_t));
+        attrib_data[i]->dataset       = dataset;
+        attrib_data[i]->datatype      = datatype;
+        attrib_data[i]->dataspace     = dataspace;
+        attrib_data[i]->current_index = i;
+        if (H5TS_thread_create(&threads[i], tts_acreate_thread, attrib_data[i]) < 0)
             TestErrPrintf("thread # %d did not start", i);
     }
 
@@ -112,7 +116,9 @@ tts_acreate(void)
 
     /* verify the correctness of the test */
     for (i = 0; i < NUM_THREADS; i++) {
-        attribute = H5Aopen(dataset, gen_name(i), H5P_DEFAULT);
+        attribute_name = gen_name(i);
+
+        attribute = H5Aopen(dataset, attribute_name, H5P_DEFAULT);
         CHECK(attribute, H5I_INVALID_HID, "H5Aopen");
 
         if (attribute < 0)
@@ -125,6 +131,8 @@ tts_acreate(void)
             status = H5Aclose(attribute);
             CHECK(status, FAIL, "H5Aclose");
         }
+
+        free(attribute_name);
     }
 
     /* close remaining resources */
@@ -136,6 +144,10 @@ tts_acreate(void)
     CHECK(status, FAIL, "H5Dclose");
     status = H5Fclose(file);
     CHECK(status, FAIL, "H5Fclose");
+
+    for (i = 0; i < NUM_THREADS; i++)
+        if (attrib_data[i])
+            free(attrib_data[i]);
 } /* end tts_acreate() */
 
 H5TS_THREAD_RETURN_TYPE
@@ -164,6 +176,8 @@ tts_acreate_thread(void *client_data)
     status = H5Aclose(attribute);
     CHECK(status, FAIL, "H5Aclose");
 
+    free(attribute_data);
+    free(attribute_name);
     return (H5TS_thread_ret_t)0;
 } /* end tts_acreate_thread() */
 

test/ttsafe_attr_vlen.c

@@ -123,6 +123,7 @@ tts_attr_vlen_thread(void H5_ATTR_UNUSED *client_data)
     hid_t       fid  = H5I_INVALID_HID; /* File ID */
     hid_t       gid  = H5I_INVALID_HID; /* Group ID */
     hid_t       aid  = H5I_INVALID_HID; /* Attribute ID */
+    hid_t       asid = H5I_INVALID_HID; /* Dataspace ID for the attribute */
     hid_t       atid = H5I_INVALID_HID; /* Datatype ID for the attribute */
     char       *string_attr_check;      /* The attribute data being read */
     const char *string_attr = "2.0";    /* The expected attribute data */
@@ -144,17 +145,28 @@ tts_attr_vlen_thread(void H5_ATTR_UNUSED *client_data)
     atid = H5Aget_type(aid);
     CHECK(atid, H5I_INVALID_HID, "H5Aget_type");
 
+    /* Get the dataspace for the attribute */
+    asid = H5Aget_space(aid);
+    CHECK(asid, H5I_INVALID_HID, "H5Aget_space");
+
     /* Read the attribute */
     ret = H5Aread(aid, atid, &string_attr_check);
     CHECK(ret, FAIL, "H5Aclose");
 
     /* Verify the attribute data is as expected */
     VERIFY_STR(string_attr_check, string_attr, "H5Aread");
 
+    /* Free the attribute data */
+    ret = H5Dvlen_reclaim(atid, asid, H5P_DEFAULT, &string_attr_check);
+    CHECK(ret, FAIL, "H5Dvlen_reclaim");
+
     /* Close IDs */
     ret = H5Aclose(aid);
     CHECK(ret, FAIL, "H5Aclose");
 
+    ret = H5Sclose(asid);
+    CHECK(ret, FAIL, "H5Aclose");
+
     ret = H5Gclose(gid);
     CHECK(ret, FAIL, "H5Aclose");