Merge remote-tracking branch 'origin/develop' into threading_updates

doxygen/aliases

@@ -235,7 +235,6 @@ ALIASES += sa_metadata_ops="\sa \li H5Pget_all_coll_metadata_ops() \li H5Pget_co
 ################################################################################
 
 ALIASES += ref_cons_semantics="<a href=\"https://portal.hdfgroup.org/display/HDF5/Enabling+a+Strict+Consistency+Semantics+Model+in+Parallel+HDF5\">Enabling a Strict Consistency Semantics Model in Parallel HDF5</a>"
-ALIASES += ref_dld_filters="<a href=\"https://portal.hdfgroup.org/display/HDF5/HDF5+Dynamically+Loaded+Filters\">HDF5 Dynamically Loaded Filters</a>"
 ALIASES += ref_file_image_ops="<a href=\"https://portal.hdfgroup.org/display/HDF5/HDF5+File+Image+Operations\">HDF5 File Image Operations</a>"
 ALIASES += ref_filter_pipe="<a href=\"https://portal.hdfgroup.org/display/HDF5/HDF5+Data+Flow+Pipeline+for+H5Dread\">Data Flow Pipeline for H5Dread()</a>"
 ALIASES += ref_group_impls="<a href=\"https://portal.hdfgroup.org/display/HDF5/Groups\">Group implementations in HDF5</a>"

doxygen/dox/UsersGuide.dox

@@ -134,6 +134,7 @@ HDF5 Release 1.14
         <li> \ref subsubsec_dataset_transfer_props
         <li> \ref subsubsec_dataset_transfer_store
         <li> \ref subsubsec_dataset_transfer_partial
+        <li> \ref subsubsec_dataset_transfer_dyn_filter
     </ul>
 \li \ref subsec_dataset_allocation
     <ul>
@@ -147,6 +148,7 @@ HDF5 Release 1.14
         <li> \ref subsubsec_dataset_filters_nbit
         <li> \ref subsubsec_dataset_filters_scale
         <li> \ref subsubsec_dataset_filters_szip
+        <li> \ref subsubsec_dataset_filters_dyn
     </ul>
 
 \ref sec_datatype

doxygen/examples/H5.format.1.1.html

@@ -5436,8 +5436,8 @@ <h4><a name="FilterMessage">Name: Data Storage - Filter Pipeline</a></h4>
             filters requested and supported by third parties.
             Filters supported by The HDF Group are documented immediately
             below.  Information on 3rd-party filters can be found at
-            <a href="/services/contributions.html#filters">
-            <code>https://support.hdfgroup.org/services/contributions.html#filters</code></a>.
+            <a href="https://portal.hdfgroup.org/documentation/hdf5-docs/registered_filter_plugins.html">
+            <code>https://portal.hdfgroup.org/documentation/hdf5-docs/registered_filter_plugins.html</code></a>.
             <a href="#Footnote1Change"><sup><small>1</small></sup></a>
             <p>
             To request a filter identifier, please contact

doxygen/examples/H5.format.2.0.html

@@ -12598,9 +12598,8 @@ <h4>
                     HDF5 Library and for filters requested and supported by third
                     parties. Filters supported by The HDF Group are documented
                     immediately below. Information on 3rd-party filters can be found at
-                    The HDF Group&rsquo;s <a
-                        href="http://www.hdfgroup.org/services/contributions.html">
-                        Contributions</a> page.
+                    The HDF Group&rsquo;s <a href="https://portal.hdfgroup.org/documentation/hdf5-docs/registered_filter_plugins.html">
+                    Registered Filters</a> page.
                 </p>
 
                 <p>
@@ -12854,9 +12853,8 @@ <h4>
                     HDF5 Library and for filters requested and supported by third
                     parties. Filters supported by The HDF Group are documented
                     immediately below. Information on 3rd-party filters can be found at
-                    The HDF Group&rsquo;s <a
-                        href="http://www.hdfgroup.org/services/contributions.html">
-                        Contributions</a> page.
+                    The HDF Group&rsquo;s <a href="https://portal.hdfgroup.org/documentation/hdf5-docs/registered_filter_plugins.html">
+                    Registered Filters</a> page.
                 </p>
 
                 <p>

doxygen/examples/H5.format.html

@@ -14226,8 +14226,8 @@ <h4><a name="FilterMessage">IV.A.2.l. The Data Storage - Filter
           Filters supported by The HDF Group are documented immediately
           below.  Information on 3rd-party filters can be found at
           The HDF Group&rsquo;s
-          <a href="http://www.hdfgroup.org/services/contributions.html">
-            Contributions</a> page.</p>
+          <a href="https://portal.hdfgroup.org/documentation/hdf5-docs/registered_filter_plugins.html">
+          Registered Filters</a> page.</p>
 
         <p>
           To request a filter identifier, please contact
@@ -14488,8 +14488,8 @@ <h4><a name="FilterMessage">IV.A.2.l. The Data Storage - Filter
           Filters supported by The HDF Group are documented immediately
           below.  Information on 3rd-party filters can be found at
           The HDF Group&rsquo;s
-          <a href="http://www.hdfgroup.org/services/contributions.html">
-            Contributions</a> page.</p>
+          <a href="https://portal.hdfgroup.org/documentation/hdf5-docs/registered_filter_plugins.html">
+          Registered Filters</a> page.</p>
 
         <p>
           To request a filter identifier, please contact

release_docs/RELEASE.txt

@@ -591,6 +591,32 @@ Bug Fixes since HDF5-1.14.0 release
 
     Library
     -------
+    - Corrected H5Soffset_simple() when offset is NULL
+
+      The reference manual states that the offset parameter of H5Soffset_simple()
+      can be set to NULL to reset the offset of a simple dataspace to 0. This
+      has never been true, and passing NULL was regarded as an error.
+
+      The library will now accept NULL for the offset parameter and will
+      correctly set the offset to zero.
+
+      Fixes HDFFV-9299
+
+    - Fixed an issue where the Subfiling VFD's context object cache could
+      grow too large
+
+      The Subfiling VFD keeps a cache of its internal context objects to
+      speed up access to a context object for a particular file, as well
+      as access to that object across multiple opens of the same file.
+      However, opening a large amount of files with the Subfiling VFD over
+      the course of an application's lifetime could cause this cache to grow
+      too large and result in the application running out of available MPI
+      communicator objects. On file close, the Subfiling VFD now simply
+      evicts context objects out of its cache and frees them. It is assumed
+      that multiple opens of a file will be a less common use case for the
+      Subfiling VFD, but this can be revisited if it proves to be an issue
+      for performance.
+
     - Fixed error when overwriting certain nested variable length types
 
       Previously, when using a datatype that included a variable length type

src/H5Dmodule.h

@@ -815,19 +815,10 @@
  * <table>
  * <caption>Data pipeline filters</caption>
  * <tr>
- * <th>Filter</th>
+ * <th>Built-in Filter</th>
  * <th>Description</th>
  * </tr>
  * <tr>
- * <td>gzip compression</td>
- * <td>Data compression using zlib.</td>
- * </tr>
- * <tr>
- * <td>Szip compression</td>
- * <td>Data compression using the Szip library. See The HDF Group website for more information
- * regarding the Szip filter.</td>
- * </tr>
- * <tr>
  * <td>N-bit compression</td>
  * <td>Data compression using an algorithm specialized for n-bit datatypes.</td>
  * </tr>
@@ -844,6 +835,19 @@
  * <tr>
  * <td>Fletcher32</td>
  * <td>Fletcher32 checksum for error-detection.</td>
+ * </tr>
+ * <tr>
+ * <th>Optional Built-in Filter</th>
+ * <th>Description</th>
+ * </tr>
+ * <tr>
+ * <td>gzip compression</td>
+ * <td>Data compression using zlib.</td>
+ * </tr>
+ * <tr>
+ * <td>szip compression</td>
+ * <td>Data compression using the szip library. The HDF Group now uses the libaec library for the szip
+filter.</td>
  * </tr>
  * </table>
  *
@@ -861,6 +865,44 @@
  * \li @see @ref subsubsec_dataset_filters_nbit
  * \li @see @ref subsubsec_dataset_filters_scale
  *
+ * \subsubsection subsubsec_dataset_transfer_dyn_filter Data Pipeline Dynamically Loaded Filters
+ * While the HDF5 “internal” compression methods work reasonably well on users’
+ * datasets, there are certain drawbacks to this implementation. First, the “internal” compression
+ * methods may not provide the optimal compression ratio, as do some newly developed or specialized
+ * compression methods. Secondly, if a data provider wants to use a “non-internal” compression for
+ * storing the data in HDF5, they have to write a filter function that uses the new compression method
+ * and then register it with the library. Data consumers of such HDF5 files will need to have the new filter
+ * function and use it with their applications to read the data, or they will need a modified version of the
+ * HDF5 Library that has the new filter as a part of the library.
+ *
+ * If a user of such data does not have a modified HDF5 Library installed on his system, command-line tools
+ * such as h5dump or h5ls will not be able to display the compressed data. Furthermore, it would be
+ * practically impossible to determine the compression method used, making the data stored in HDF5
+ * useless.
+ *
+ * It is clear that the internal HDF5 filter mechanism, while extensible, does not work well with third-party
+ * filters. It would be a maintenance nightmare to keep adding and supporting new compression methods
+ * in HDF5. For any set of HDF5 “internal” filters, there always will be data with which the “internal”
+filters
+ * will not achieve the optimal performance needed to address data I/O and storage problems. Thus the
+ * internal HDF5 filter mechanism is enhanced to address the issues discussed above.
+ *
+ * We have a feature of HDF5 called “dynamically loaded filters in HDF5.” This feature
+ * makes the HDF5 third-party filters available to an application at runtime. The third-party HDF5 filter
+ * function has to be a part of the HDF5 filter plugin installed on the system as a shared library or DLL.
+ *
+ * To use a third-party filter an HDF5 application should call the #H5Pset_filter function when setting the
+ * filter pipeline for a dataset creation property. The HDF5 Library will register the filter with the library
+ * and the filter will be applied when data is written to the file.
+ *
+ * When an application reads data compressed with a third-party HDF5 filter, the HDF5 Library will search
+ * for the required filter plugin, register the filter with the library (if the filter function is not
+registered) and
+ * apply it to the data on the read operation.
+ *
+ * For more information,
+ * \li @see @ref sec_filter_plugins
+ *
  * \subsubsection subsubsec_dataset_transfer_drive File Drivers
  * I/O is performed by the HDF5 virtual file layer. The file driver interface writes and reads blocks
  * of data; each driver module implements the interface using different I/O mechanisms. The table
@@ -2685,8 +2727,25 @@ allocated if necessary.
  * and minimum values, and they will get a much larger minimum-bits (poor compression)
  * </li></ol>
  *
- * \subsubsection subsubsec_dataset_filters_szip Using the Szip Filter
- * See The HDF Group website for further information regarding the Szip filter.
+ * \subsubsection subsubsec_dataset_filters_szip Using the SZip Filter
+ * See The HDF Group website for further information regarding the SZip filter.
+ *
+ * \subsubsection subsubsec_dataset_filters_dyn Using Dynamically-Loadable Filters
+ * \see \ref sec_filter_plugins for further information regarding the dynamically-loadable filters.
+ *
+ * HDF has a filter plugin repository of useful third-party plugins that can used
+ * <table>
+ * <tr><th>Filter</th><th>SetFilter Params</th></tr>
+ * <tr><td>BLOSC</td><td>UD=32001,0,0</td></tr>
+ * <tr><td>BSHUF</td><td>UD=32004,0,0</td></tr>
+ * <tr><td>BZIP2</td><td>UD=307,0,1,9</td></tr>
+ * <tr><td>JPEG</td><td>UD=32019,0,4,q,c,r,t</td></tr>
+ * <tr><td>LZ4 </td><td>UD=32004,0,1,3</td></tr>
+ * <tr><td>LZF</td><td>UD=32000,1,3,0,0,0</td></tr>
+ * <tr><td>SZ</td><td>UD=32017,1,5,2,7,20,40,0</td></tr>
+ * <tr><td>ZFP</td><td>UD=32013,1,0,0</td></tr>
+ * <tr><td>ZSTD</td><td>UD=32015,0,0</td></tr>
+ * </table>
  *
  * Previous Chapter \ref sec_group - Next Chapter \ref sec_datatype
  *

src/H5FDmirror.h

@@ -35,7 +35,7 @@
 #define H5FD_MIRROR_CURR_FAPL_T_VERSION 1
 
 /** Max size of the remote_ip array in H5FD_mirror_fapl_t */
-#define H5FD_MIRROR_MAX_IP_LEN 32
+#define H5FD_MIRROR_MAX_IP_LEN 45 /* Max size of an IPv4-mapped IPv6 address */
 
 /**
  *\struct H5FD_mirror_fapl_t

src/H5FDsubfiling/H5FDsubfiling.c

@@ -1358,6 +1358,9 @@ H5FD__subfiling_close_int(H5FD_subfiling_t *file_ptr)
     H5MM_free(file_ptr->file_dir);
     file_ptr->file_dir = NULL;
 
+    if (file_ptr->context_id >= 0 && H5_free_subfiling_object(file_ptr->context_id) < 0)
+        H5_SUBFILING_DONE_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free subfiling context object");
+
     /* Release the file info */
     file_ptr = H5FL_FREE(H5FD_subfiling_t, file_ptr);
 

src/H5FDsubfiling/H5subfiling_common.c

@@ -51,7 +51,6 @@ static H5TS_mutex_t subfiling_log_mutex = H5TS_MUTEX_INITIALIZER;
 #define DEFAULT_TOPOLOGY_CACHE_SIZE 4
 #define DEFAULT_FILE_MAP_ENTRIES    8
 
-static herr_t H5_free_subfiling_object(int64_t object_id);
 static herr_t H5_free_subfiling_object_int(subfiling_context_t *sf_context);
 static herr_t H5_free_subfiling_topology(sf_topology_t *topology);
 
@@ -283,18 +282,25 @@ H5_get_subfiling_object(int64_t object_id)
  * Purpose:     Frees the underlying subfiling object for a given subfiling
  *              object ID.
  *
- *              NOTE: Currently we assume that all created subfiling
- *              objects are cached in the (very simple) context/topology
- *              cache until application exit, so the only time a subfiling
- *              object should be freed by this routine is if something
- *              fails right after creating one. Otherwise, the internal
- *              indexing for the relevant cache will be invalid.
+ *              NOTE: Because we want to avoid the potentially large
+ *              overhead of determining the application topology on every
+ *              file open, we currently assume that all created subfiling
+ *              topology objects are cached in the (very simple) topology
+ *              cache until application exit. This allows us to quickly
+ *              find and assign a cached topology object to a subfiling
+ *              context object for a file when opened. Therefore, a
+ *              subfiling topology object should (currently) only ever be
+ *              freed by this routine if a function fails right after
+ *              creating a topology object. Otherwise, the internal
+ *              indexing for the topology cache will be invalid and we will
+ *              either leak memory or assign invalid topology pointers to
+ *              subfiling context objects after that point.
  *
  * Return:      Non-negative on success/Negative on failure
  *
  *-------------------------------------------------------------------------
  */
-static herr_t
+herr_t
 H5_free_subfiling_object(int64_t object_id)
 {
     int64_t obj_type  = (object_id >> 32) & 0x0FFFF;
@@ -308,30 +314,31 @@ H5_free_subfiling_object(int64_t object_id)
                                     "couldn't get subfiling context for subfiling object ID");
 
         if (H5_free_subfiling_object_int(sf_context) < 0)
-            H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "couldn't free subfiling object");
+            H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "couldn't free subfiling context object");
 
         assert(sf_context_cache_num_entries > 0);
         assert(sf_context == sf_context_cache[sf_context_cache_num_entries - 1]);
         sf_context_cache[sf_context_cache_num_entries - 1] = NULL;
         sf_context_cache_num_entries--;
     }
-    else {
+    else if (obj_type == SF_TOPOLOGY) {
         sf_topology_t *sf_topology;
 
-        assert(obj_type == SF_TOPOLOGY);
-
         if (NULL == (sf_topology = H5_get_subfiling_object(object_id)))
             H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL,
                                     "couldn't get subfiling context for subfiling object ID");
 
         if (H5_free_subfiling_topology(sf_topology) < 0)
-            H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "couldn't free subfiling topology");
+            H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "couldn't free subfiling topology object");
 
         assert(sf_topology_cache_num_entries > 0);
         assert(sf_topology == sf_topology_cache[sf_topology_cache_num_entries - 1]);
         sf_topology_cache[sf_topology_cache_num_entries - 1] = NULL;
         sf_topology_cache_num_entries--;
     }
+    else
+        H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL,
+                                "couldn't free subfiling object - invalid object type");
 
 done:
     H5_SUBFILING_FUNC_LEAVE;

src/H5FDsubfiling/H5subfiling_common.h

@@ -269,6 +269,7 @@ H5_DLL herr_t H5_close_subfiles(int64_t subfiling_context_id, MPI_Comm file_comm
 
 H5_DLL int64_t H5_new_subfiling_object_id(sf_obj_type_t obj_type);
 H5_DLL void   *H5_get_subfiling_object(int64_t object_id);
+H5_DLL herr_t  H5_free_subfiling_object(int64_t object_id);
 H5_DLL herr_t  H5_get_subfiling_config_from_file(FILE *config_file, int64_t *stripe_size,
                                                  int64_t *num_subfiles);
 H5_DLL herr_t  H5_resolve_pathname(const char *filepath, MPI_Comm comm, char **resolved_filepath);