Integrate shared chunk cache with get_defined and erase operations.

Various other fixes/enhancements.

src/H5Dchunk.c

@@ -56,6 +56,7 @@
 #include "H5MMprivate.h" /* Memory management                        */
 #include "H5MFprivate.h" /* File memory management                   */
 #include "H5PBprivate.h" /* Page Buffer	                             */
+#include "H5SCprivate.h" /* Shared chunk cache                       */
 #include "H5SLprivate.h" /* Skip Lists                               */
 #include "H5VMprivate.h" /* Vector and array functions               */
 
@@ -8281,22 +8282,28 @@ H5D__chunk_verify_offset(const H5D_t *dset, const hsize_t *offset)
  *-------------------------------------------------------------------------
  */
 herr_t
-H5D__get_struct_chunk_info(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *space,
+H5D__get_struct_chunk_info(H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space,
                            hsize_t H5_ATTR_UNUSED chunk_idx, hsize_t H5_ATTR_UNUSED *offset,
                            H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info, haddr_t H5_ATTR_UNUSED *addr,
                            hsize_t H5_ATTR_UNUSED *chunk_size)
 {
     herr_t ret_value = SUCCEED; /* Return value */
 
-    FUNC_ENTER_PACKAGE_NOERR
+    FUNC_ENTER_PACKAGE
 
     assert(dset);
     assert(dset->shared);
     assert(space);
 
+    /* Flush the dataset's cached chunks out to disk, to make certain the size is correct later */
+    /* It should be possible to optimize this in the future by only flushing the target chunk, and later directly looking up the target chunk instead of iterating, and potentially avoiding the flush and/or index query completely if the shared chunk cache has all the needed information needed. For now, just mirror the previous algorithm for legacy chunks. */
+    if (H5SC_flush_dset(H5F_SHARED_CACHE(dset->oloc.file), dset, false) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "cannot flush shared chunk cache for dataset");
+
     /* TBD: go get structured chunk information using chunk index */
     /* FOR NOW: just return success */
 
+done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5D__get_struct_chunk_info() */
 
@@ -8312,22 +8319,28 @@ H5D__get_struct_chunk_info(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR
  *-------------------------------------------------------------------------
  */
 herr_t
-H5D__get_struct_chunk_info_by_coord(const H5D_t H5_ATTR_UNUSED *dset, const hsize_t H5_ATTR_UNUSED *offset,
+H5D__get_struct_chunk_info_by_coord(H5D_t *dset, const hsize_t H5_ATTR_UNUSED *offset,
                                     H5D_struct_chunk_info_t H5_ATTR_UNUSED *chunk_info,
                                     haddr_t H5_ATTR_UNUSED *addr, hsize_t H5_ATTR_UNUSED *chunk_size)
 {
     herr_t ret_value = SUCCEED; /* Return value */
 
-    FUNC_ENTER_PACKAGE_NOERR
+    FUNC_ENTER_PACKAGE
 
     /* Check args */
     assert(dset);
     assert(dset->shared);
     assert(offset);
 
+    /* Flush the dataset's cached chunks out to disk, to make certain the size is correct later */
+    /* It should be possible to optimize this in the future by only flushing the target chunk, and later directly looking up the target chunk instead of iterating, and potentially avoiding the flush and/or index query completely if the shared chunk cache has all the needed information needed. For now, just mirror the previous algorithm for legacy chunks. */
+    if (H5SC_flush_dset(H5F_SHARED_CACHE(dset->oloc.file), dset, false) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "cannot flush shared chunk cache for dataset");
+
     /* TBD: go get structured chunk information using chunk coordinates */
     /* FOR NOW: just return success */
 
+done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5D__get_struct_chunk_info_by_coord() */
 
@@ -8341,19 +8354,24 @@ H5D__get_struct_chunk_info_by_coord(const H5D_t H5_ATTR_UNUSED *dset, const hsiz
  *-------------------------------------------------------------------------
  */
 herr_t
-H5D__struct_chunk_iter(H5D_t H5_ATTR_UNUSED *dset, H5D_struct_chunk_iter_op_t H5_ATTR_UNUSED op,
+H5D__struct_chunk_iter(H5D_t *dset, H5D_struct_chunk_iter_op_t H5_ATTR_UNUSED op,
                        void H5_ATTR_UNUSED *op_data)
 {
     herr_t ret_value = SUCCEED; /* Return value */
 
-    FUNC_ENTER_PACKAGE_NOERR
+    FUNC_ENTER_PACKAGE
 
     /* Check args */
     assert(dset);
     assert(dset->shared);
 
+    /* Flush the dataset's cached chunks out to disk, to make certain the size is correct later */
+    if (H5SC_flush_dset(H5F_SHARED_CACHE(dset->oloc.file), dset, false) < 0)
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "cannot flush shared chunk cache for dataset");
+
     /* TBD: iterate over all the structured chunks in the dataset */
     /* FOR NOW: just return success */
 
+done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5D__chunk_iter() */

src/H5Dint.c

@@ -2037,7 +2037,7 @@ H5D_close(H5D_t *dataset)
 
         /* Evict the dataset's entries in the shared chunk cache */
         if (dataset->shared->layout.sc_ops &&
-            H5SC_flush_dset(H5F_get_shared_cache(dataset->oloc.file), dataset, true) < 0)
+            H5SC_flush_dset(H5F_SHARED_CACHE(dataset->oloc.file), dataset, true) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL,
                         "unable to evict dataset's entries in shared chunk cache");
 
@@ -3224,7 +3224,7 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size)
 
         /* Notify the shared chunk cache that the extent has changed */
         if (dset->shared->layout.sc_ops)
-            if (H5SC_set_extent_notify(H5F_get_shared_cache(dset->oloc.file), dset, curr_dims) < 0)
+            if (H5SC_set_extent_notify(H5F_SHARED_CACHE(dset->oloc.file), dset, curr_dims) < 0)
                 HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
                             "unable to notify shared chunk cache of extent change");
 
@@ -3303,7 +3303,7 @@ H5D__flush_real(H5D_t *dataset)
 
         /* Flush the dataset's entries in the shared chunk cache */
         if (dataset->shared->layout.sc_ops &&
-            H5SC_flush_dset(H5F_get_shared_cache(dataset->oloc.file), dataset, false) < 0)
+            H5SC_flush_dset(H5F_SHARED_CACHE(dataset->oloc.file), dataset, false) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush shared chunk cache");
     }
 
@@ -4090,24 +4090,24 @@ H5D_get_dcpl_id(const H5D_obj_create_t *d)
  *-------------------------------------------------------------------------
  */
 hid_t
-H5D__get_defined(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t *fspace)
+H5D__get_defined(H5D_t *dset, const H5S_t *fspace)
 {
     H5S_t *space     = NULL;
     hid_t  ret_value = H5I_INVALID_HID;
 
     FUNC_ENTER_PACKAGE
 
-    /* TBD:
-        if (dset->shared->layout.type == H5D_SPARSE_CHUNK)
-            call routine to get defined elements
-        else
-            if (NULL == (space = H5S_copy(fspace, false, true)))
-                HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace");
-     */
-
-    /* FOR NOW: return copy of fspace */
-    if (NULL == (space = H5S_copy(fspace, false, true)))
-        HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace");
+    /* Check for shared chunk cache */
+    if (dset->shared->layout.sc_ops) {
+        /* Forward to SCC layer */
+        if (NULL == (space = H5SC_get_defined(H5F_SHARED_CACHE(dset->oloc.file), dset, fspace)))
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get defined values");
+    }
+    else
+        /* No SCC support, so no support for defined values, entire dataset is defined */
+        /* Clip to dataset extent? -NAF */
+        if (NULL == (space = H5S_copy(fspace, false, true)))
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get dataspace");
 
     /* Create an ID */
     if ((ret_value = H5I_register(H5I_DATASPACE, space, true)) < 0)
@@ -4134,19 +4134,22 @@ H5D__get_defined(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t *fspace)
  *-------------------------------------------------------------------------
  */
 herr_t
-H5D__erase(const H5D_t H5_ATTR_UNUSED *dset, const H5S_t H5_ATTR_UNUSED *fspace)
+H5D__erase(H5D_t *dset, const H5S_t *fspace)
 {
     herr_t ret_value = SUCCEED; /* Return value */
 
-    FUNC_ENTER_PACKAGE_NOERR
+    FUNC_ENTER_PACKAGE
 
-    /* TBD:
-        if (dset->shared->layout.type == H5D_SPARSE_CHUNK)
-            call routine to delete elements
-        else
-            return error
-     */
-    /* FOR NOW: just return success */
+    /* Check for shared chunk cache */
+    if (dset->shared->layout.sc_ops) {
+        /* Forward to SCC layer */
+        if (H5SC_erase(H5F_SHARED_CACHE(dset->oloc.file), dset, fspace) < 0)
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "unable to get defined values");
+    }
+    else
+        /* No SCC support, so no support for defined values and hence cannot erase */
+        HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "dataset does not support erasing values");
 
+done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* H5D__erase() */

src/H5Dio.c

@@ -467,8 +467,8 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info)
 
     /* Make shared chunk cache read call if appropriate */
     if (any_scc) {
-        assert(H5F_get_shared_cache(dset_info[0].dset->oloc.file));
-        if (H5SC_read(H5F_get_shared_cache(dset_info[0].dset->oloc.file), count, dset_info) < 0)
+        assert(H5F_SHARED_CACHE(dset_info[0].dset->oloc.file));
+        if (H5SC_read(H5F_SHARED_CACHE(dset_info[0].dset->oloc.file), count, dset_info) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "read through shared chunk cache failed");
     }
 
@@ -917,8 +917,8 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info)
 
     /* Make shared chunk cache write call if appropriate */
     if (any_scc) {
-        assert(H5F_get_shared_cache(dset_info[0].dset->oloc.file));
-        if (H5SC_write(H5F_get_shared_cache(dset_info[0].dset->oloc.file), count, dset_info) < 0)
+        assert(H5F_SHARED_CACHE(dset_info[0].dset->oloc.file));
+        if (H5SC_write(H5F_SHARED_CACHE(dset_info[0].dset->oloc.file), count, dset_info) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "write through shared chunk cache failed");
     }
 

src/H5Dpkg.h

@@ -782,17 +782,17 @@ H5_DLL herr_t H5D__chunk_stats(const H5D_t *dset, bool headers);
 #endif /* H5D_CHUNK_DEBUG */
 
 /* Functions that operate on H5D_SPARSE_CHUNK storage */
-H5_DLL hid_t  H5D__get_defined(const H5D_t *dset, const H5S_t *fspace);
-H5_DLL herr_t H5D__erase(const H5D_t *dset, const H5S_t *fspace);
+H5_DLL hid_t  H5D__get_defined(H5D_t *dset, const H5S_t *fspace);
+H5_DLL herr_t H5D__erase(H5D_t *dset, const H5S_t *fspace);
 H5_DLL herr_t H5D__write_struct_chunk_direct(H5D_t *dset, hsize_t *offset,
                                              H5D_struct_chunk_info_t *chunk_info, void *buf[]);
 H5_DLL herr_t H5D__read_struct_chunk_direct(const H5D_t *dset, hsize_t *offset,
                                             H5D_struct_chunk_info_t *chunk_info, void *buf[]);
-H5_DLL herr_t H5D__get_struct_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space,
+H5_DLL herr_t H5D__get_struct_chunk_info(H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space,
                                          hsize_t chunk_idx, hsize_t *offset,
                                          H5D_struct_chunk_info_t *chunk_info, haddr_t *addr,
                                          hsize_t *chunk_size);
-H5_DLL herr_t H5D__get_struct_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset,
+H5_DLL herr_t H5D__get_struct_chunk_info_by_coord(H5D_t *dset, const hsize_t *offset,
                                                   H5D_struct_chunk_info_t *chunk_info, haddr_t *addr,
                                                   hsize_t *chunk_size);
 H5_DLL herr_t H5D__struct_chunk_iter(H5D_t *dset, H5D_struct_chunk_iter_op_t cb, void *op_data);

src/H5SC.c

@@ -148,6 +148,7 @@ H5SC_flush_dset(H5SC_t *cache, H5D_t *dset, bool evict)
 
     assert(cache);
     assert(dset);
+    assert(dset->shared->layout.sc_ops);
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -224,6 +225,7 @@ H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void *
 
     assert(cache);
     assert(dset);
+    assert(dset->shared->layout.sc_ops);
     assert(offset);
     assert(buf);
     assert(buf_size);
@@ -252,13 +254,81 @@ H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void
 
     assert(cache);
     assert(dset);
+    assert(dset->shared->layout.sc_ops);
     assert(offset);
     assert(buf);
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5SC_direct_chunk_write() */
 
+/*-------------------------------------------------------------------------
+ * Function: H5SC_get_defined
+ *
+ * Purpose:  Returns a copy of file_space with only elements selected that are both selected in file_space and defined in dset. If file_space uses a point selection, the ordering of selected points will be preserved in the returned dataspace.
+ *
+ * Return:   SUCCEED on success, FAIL on failure
+ *-------------------------------------------------------------------------
+ */
+H5S_t *
+H5SC_get_defined(H5SC_t *cache, H5D_t *dset, const H5S_t *file_space)
+{
+    H5S_t *defined = NULL;
+    H5S_t *ret_value = NULL;
+
+    FUNC_ENTER_NOAPI(NULL)
+
+    assert(cache);
+    assert(dset);
+    assert(dset->shared->layout.sc_ops);
+    assert(file_space);
+
+    /* FOR NOW: just return copy of file_space */
+    if (NULL == (defined = H5S_copy(file_space, false, true)))
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to copy dataspace");
+
+    /* Set return value */
+    ret_value = defined;
+    defined = NULL;
+
+done:
+    if (defined) {
+        assert(!ret_value);
+        if (H5S_close(defined) < 0)
+            HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, NULL, "unable to release dataspace");
+    }
+
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_get_defined() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5SC_erase
+ *
+ * Purpose:  Causes the elements selected in file_space to become undefined in dset. If dset does not support tracking defined elements, returns an error.
+ *
+ * Return:   SUCCEED on success, FAIL on failure
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5SC_erase(H5SC_t *cache, H5D_t *dset, const H5S_t *file_space)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    assert(cache);
+    assert(dset);
+    assert(dset->shared->layout.sc_ops);
+    assert(file_space);
+
+    /* Check for support for erasing values */
+    if (!dset->shared->layout.sc_ops->erase_values)
+        HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "dataset does not support erasing values");
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_erase() */
+
 /*-------------------------------------------------------------------------
  * Function: H5SC_set_extent_notify
  *
@@ -270,14 +340,15 @@ H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, const hsize_t *offset, void
  *-------------------------------------------------------------------------
  */
 herr_t
-H5SC_set_extent_notify(H5SC_t *cache, H5D_t *dset, hsize_t *old_dims)
+H5SC_set_extent_notify(H5SC_t *cache, H5D_t *dset, const hsize_t *old_dims)
 {
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI(FAIL)
 
     assert(cache);
     assert(dset);
+    assert(dset->shared->layout.sc_ops);
     assert(old_dims);
 
 done: