Fixed a cache assert with too-large metadata objects (#4231)

If the library tries to load a metadata object that is above the
library's hard-coded limits, the size will trip an assert in debug
builds. In HDF5 1.14.4, this can happen if you create a very large
number of links in an old-style group that uses local heaps.

The library will now emit a normal error when it tries to load a
metadata object that is too large.

Partially addresses GitHub #3762

release_docs/RELEASE.txt

@@ -693,6 +693,18 @@ Bug Fixes since HDF5-1.14.0 release
 
     Library
     -------
+    - Fixed a cache assert with too-large metadata objects
+
+      If the library tries to load a metadata object that is above the
+      library's hard-coded limits, the size will trip an assert in debug
+      builds. In HDF5 1.14.4, this can happen if you create a very large
+      number of links in an old-style group that uses local heaps.
+
+      The library will now emit a normal error when it tries to load a
+      metadata object that is too large.
+
+      Partially addresses GitHub #3762
+
     - Fixed an issue with the Subfiling VFD and multiple opens of a
       file
 

src/H5Centry.c

@@ -1216,10 +1216,9 @@ H5C__load_entry(H5F_t *f,
 
     assert((dirty == false) || (type->id == 5 || type->id == 6));
 
-    entry->cache_ptr = f->shared->cache;
-    entry->addr      = addr;
-    entry->size      = len;
-    assert(entry->size < H5C_MAX_ENTRY_SIZE);
+    entry->cache_ptr        = f->shared->cache;
+    entry->addr             = addr;
+    entry->size             = len;
     entry->image_ptr        = image;
     entry->image_up_to_date = !dirty;
     entry->type             = type;
@@ -1289,6 +1288,14 @@ H5C__load_entry(H5F_t *f,
 
     H5C__RESET_CACHE_ENTRY_STATS(entry);
 
+    /* This is a temporary fix for a problem identified in GitHub #3762, where
+     * it looks like a local heap entry can grow to a size that is larger
+     * than the metadata cache will allow. This doesn't fix the underlying
+     * problem, but it at least prevents the library from crashing.
+     */
+    if (entry->size >= H5C_MAX_ENTRY_SIZE)
+        HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "cache entry size is too large");
+
     ret_value = thing;
 
 done: