Add skeleton implementation of shared chunk cache (#5156)

src/CMakeLists.txt

@@ -574,6 +574,15 @@ set (H5S_HDRS
 IDE_GENERATED_PROPERTIES ("H5S" "${H5S_HDRS}" "${H5S_SOURCES}" )
 
 
+set (H5SC_SOURCES
+    ${HDF5_SRC_DIR}/H5SC.c
+)
+
+set (H5SC_HDRS
+)
+IDE_GENERATED_PROPERTIES ("H5SC" "${H5SC_HDRS}" "${H5SC_SOURCES}" )
+
+
 set (H5SL_SOURCES
     ${HDF5_SRC_DIR}/H5SL.c
 )
@@ -775,6 +784,7 @@ set (H5_MODULE_HEADERS
     ${HDF5_SRC_DIR}/H5Rmodule.h
     ${HDF5_SRC_DIR}/H5RSmodule.h
     ${HDF5_SRC_DIR}/H5Smodule.h
+    ${HDF5_SRC_DIR}/H5SCmodule.h
     ${HDF5_SRC_DIR}/H5SLmodule.h
     ${HDF5_SRC_DIR}/H5SMmodule.h
     ${HDF5_SRC_DIR}/H5Tmodule.h
@@ -973,6 +983,9 @@ set (H5_PRIVATE_HEADERS
     ${HDF5_SRC_DIR}/H5Spkg.h
     ${HDF5_SRC_DIR}/H5Sprivate.h
 
+    ${HDF5_SRC_DIR}/H5SCpkg.h
+    ${HDF5_SRC_DIR}/H5SCprivate.h
+
     ${HDF5_SRC_DIR}/H5SLprivate.h
 
     ${HDF5_SRC_DIR}/H5SMpkg.h

src/H5Dpkg.h

@@ -115,7 +115,7 @@
 /* Package Private Typedefs */
 /****************************/
 
-/* Typedef for datatype information for raw data I/O operation */
+/* Typedef for datatype information for a single dataset in a raw data I/O operation */
 typedef struct H5D_type_info_t {
     /* Initial values */
     const H5T_t *mem_type;  /* Pointer to memory datatype */
@@ -134,6 +134,20 @@ typedef struct H5D_type_info_t {
     size_t                   request_nelmts; /* Requested strip mine */
 } H5D_type_info_t;
 
+/* Typedef for datatype information for all datasets in a raw data I/O operation */
+struct H5D_io_type_info_t {
+    uint8_t            *tconv_buf;           /* Datatype conv buffer */
+    bool                tconv_buf_allocated; /* Whether the type conversion buffer was allocated */
+    size_t              tconv_buf_size;      /* Size of type conversion buffer */
+    uint8_t            *bkg_buf;             /* Background buffer */
+    bool                bkg_buf_allocated;   /* Whether the background buffer was allocated */
+    size_t              bkg_buf_size;        /* Size of background buffer */
+    H5T_vlen_buf_info_t vlen_buf_info;       /* Vlen data buffer and info */
+    bool must_fill_bkg; /* Whether any datasets need a background buffer filled with destination contents */
+    bool may_use_in_place_tconv; /* Whether datasets in this I/O could potentially use in-place type
+                                       conversion if the type sizes are compatible with it */
+};
+
 /* Forward declaration of structs used below */
 struct H5D_io_info_t;
 struct H5D_dset_io_info_t;
@@ -259,7 +273,7 @@ typedef struct H5D_piece_info_t {
 } H5D_piece_info_t;
 
 /* I/O info for a single dataset */
-typedef struct H5D_dset_io_info_t {
+struct H5D_dset_io_info_t {
     H5D_t                  *dset;       /* Pointer to dataset being operated on */
     H5D_storage_t          *store;      /* Dataset storage info */
     H5D_layout_ops_t        layout_ops; /* Dataset layout I/O operation function pointers */
@@ -281,7 +295,7 @@ typedef struct H5D_dset_io_info_t {
     const H5T_t    *mem_type; /* memory datatype */
     H5D_type_info_t type_info;
     bool            skip_io; /* Whether to skip I/O for this dataset */
-} H5D_dset_io_info_t;
+};
 
 /* I/O info for entire I/O operation */
 typedef struct H5D_io_info_t {

src/H5Dprivate.h

@@ -138,6 +138,10 @@
 typedef struct H5D_t            H5D_t;
 typedef struct H5D_obj_create_t H5D_obj_create_t;
 
+/* Other forward declarations of structs needed by this file */
+typedef struct H5D_io_type_info_t H5D_io_type_info_t;
+typedef struct H5D_dset_io_info_t H5D_dset_io_info_t;
+
 /* Typedef for cached dataset creation property list information */
 typedef struct H5D_dcpl_cache_t {
     H5O_fill_t  fill;  /* Fill value info (H5D_CRT_FILL_VALUE_NAME) */

src/H5SC.c

@@ -0,0 +1,278 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the root of the source code       *
+ * distribution tree, or in https://www.hdfgroup.org/licenses.               *
+ * If you do not have access to either file, you may request a copy from     *
+ * [email protected].                                                        *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/****************/
+/* Module Setup */
+/****************/
+
+#include "H5SCmodule.h" /* This source code file is part of the H5SC module */
+#define H5D_FRIEND      /* Suppress error about including H5Dpkg */
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h"   /* Generic Functions            */
+#include "H5Dpkg.h"      /* Datasets                     */
+#include "H5Eprivate.h"  /* Error handling               */
+#include "H5Fprivate.h"  /* Files                        */
+#include "H5MMprivate.h" /* Memory management            */
+#include "H5SCpkg.h"     /* Shared chunk cache           */
+
+/****************/
+/* Local Macros */
+/****************/
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+/* Package initialization variable */
+bool H5_PKG_INIT_VAR = false;
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/*-------------------------------------------------------------------------
+ * Function: H5SC_create
+ *
+ * Purpose:  Creates a new, empty shared chunk cache.
+ *
+ * Return:   Pointer to newly created cache on success, NULL on failure
+ *-------------------------------------------------------------------------
+ */
+H5SC_t *
+H5SC_create(H5F_t *file, hid_t fapl_id)
+{
+    H5SC_t *cache     = NULL;
+    H5SC_t *ret_value = NULL;
+
+    FUNC_ENTER_NOAPI(NULL)
+
+    assert(file);
+
+    /* Allocated cache struct */
+    if (NULL == (cache = H5MM_malloc(sizeof(H5SC_t))))
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate buffer for shared chunk cache");
+
+    /* Success */
+    ret_value = cache;
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_create() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5SC_destroy
+ *
+ * Purpose:  Destroys a shared chunk cache, freeing all data used. Does
+ *           not flush chunks.
+ *
+ * Return:   SUCCEED on success, FAIL on failure
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5SC_destroy(H5SC_t *cache)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    assert(cache);
+
+    H5MM_free(cache);
+    cache = NULL;
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_destroy() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5SC_flush
+ *
+ * Purpose:  Flushes all cached data from a shared chunk cache.
+ *
+ * Return:   SUCCEED on success, FAIL on failure
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5SC_flush(H5SC_t *cache)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    assert(cache);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_flush() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5SC_flush_dset
+ *
+ * Purpose:  Flushes all data cached for a single dataset. If evict is
+ *           true, also evicts all cached data.
+ *
+ * Return:   SUCCEED on success, FAIL on failure
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5SC_flush_dset(H5SC_t *cache, H5D_t *dset, bool evict)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    assert(cache);
+    assert(dset);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_flush_dset() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5SC_read
+ *
+ * Purpose:  Reads raw data through a shared chunk cache.
+ *
+ * Return:   SUCCEED on success, FAIL on failure
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5SC_read(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info, H5D_io_type_info_t *io_type_info)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    assert(cache);
+    assert(count == 0 || dset_info);
+    assert(io_type_info);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_read() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5SC_write
+ *
+ * Purpose:  Writes raw data through a shared chunk cache.
+ *
+ * Return:   SUCCEED on success, FAIL on failure
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5SC_write(H5SC_t *cache, size_t count, H5D_dset_io_info_t *dset_info, H5D_io_type_info_t *io_type_info)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    assert(cache);
+    assert(count == 0 || dset_info);
+    assert(io_type_info);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_write() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5SC_direct_chunk_read
+ *
+ * Purpose:  Reads the chunk that starts at coordinates give by offset
+ *           directly from disk to buf, without any decoding or
+ *           conversion. First flushes that chunk if it is dirty in the
+ *           cache.
+ *
+ * Return:   SUCCEED on success, FAIL on failure
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5SC_direct_chunk_read(H5SC_t *cache, H5D_t *dset, hsize_t *offset, void *buf)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    assert(cache);
+    assert(dset);
+    assert(offset);
+    assert(buf);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_direct_chunk_read() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5SC_direct_chunk_write
+ *
+ * Purpose:  Writes the chunk that starts at coordinates give by offset
+ *           directly from buf to disk, without any decoding or
+ *           conversion. First evicts that chunk from cache if it is
+ *           present.
+ *
+ * Return:   SUCCEED on success, FAIL on failure
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5SC_direct_chunk_write(H5SC_t *cache, H5D_t *dset, hsize_t *offset, const void *buf)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    assert(cache);
+    assert(dset);
+    assert(offset);
+    assert(buf);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_direct_chunk_write() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5SC_set_extent_notify
+ *
+ * Purpose:  Called after H5Dset_extent() has been called for a dataset,
+ *           so the cache can recompute chunk indices, delete chunks,
+ *           clear unused sections of chunks, etc.
+ *
+ * Return:   SUCCEED on success, FAIL on failure
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5SC_set_extent_notify(H5SC_t *cache, H5D_t *dset, hsize_t *old_dims)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    assert(cache);
+    assert(dset);
+    assert(old_dims);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5SC_set_extent_notify() */