Skip to content

Commit

Permalink
Backport H5Dchunk_iter to 1.12 branch (#1970)
Browse files Browse the repository at this point in the history 
* Revert "Revert H5Dchunk_iter changes in hdf5_1_12_1 (#733)"

This reverts commit 10abe9a.

* Apply clang-format

* Reincorporate spelling fix from #1166

* Incorporate H5Dchunk_iter changes from #161

* Backport to 1.12: Adds a quick for for some egregious chunk_info badness (#722)

* Backport to 1.12: Converts testhdf5 macros to h5test macros in chunk_info.c (#1820)

The two macro schemes were not designed to work together. Also
quiets some MSVC warnings about comparing pointers and integers.

* Backport to 1.12: H5Dchunk_iter now passes offsets in units of dataset elements, fix #1419 (#1969)

* H5Dchunk_iter now passes chunk dimension scaled offsets, fix #1419

* Update docs for H5Dchunk_iter, H5Dget_chunk_info*

Modified description for `H5Dchunk_iter`, `H5Dget_chunk_info`, and `H5Dget_chunk_info_by_coord` to the following

 * offset          Logical position of the chunk’s first element in units of dataset elements
 * filter_mask  Bitmask indicating the filters used when the chunk was written
 * size             Chunk size in bytes, 0 if the chunk does not exist

* Sync H5Dchunk_iter documentation with develop

* Remove H5VL_DATASET_WAIT references from 1.12

* Backport to 1.12 #161, #1474, review comments

Co-authored-by: Dana Robinson <[email protected]>
  • Loading branch information
@mkitti @derobins
mkitti and derobins authored Aug 16, 2022
1 parent d9247ec commit 0c1898d
Show file tree
Hide file tree
Showing 8 changed files with 441 additions and 69 deletions.
1 change: 1 addition & 0 deletions bin/trace
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ $Source = "";
"H5D_vds_view_t" => "Dv",
"H5FD_mpio_xfer_t" => "Dt",
"H5FD_splitter_vfd_config_t" => "Dr",
"H5D_chunk_iter_op_t" => "x",
"herr_t" => "e",
"H5E_direction_t" => "Ed",
"H5E_error_t" => "Ee",
Expand Down
74 changes: 74 additions & 0 deletions src/H5D.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1126,3 +1126,77 @@ H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filte
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Dget_chunk_info_by_coord() */

/*-------------------------------------------------------------------------
* Function: H5Dchunk_iter
*
* Purpose: Iterates over all chunks in dataset with given callback and user data.
*
* Parameters:
* hid_t dset_id; IN: Chunked dataset ID
* hid_t dxpl_id; IN: Dataset transfer property list ID
* H5D_chunk_iter_op_t cb IN: User callback function, called for every chunk.
* void *op_data IN/OUT: Optional user data passed on to user callback.
*
* Callback information:
* H5D_chunk_iter_op_t is defined as:
*
* typedef int (*H5D_chunk_iter_op_t)(
* const hsize_t *offset,
* uint32_t filter_mask,
* haddr_t addr,
* uint32_t size,
* void *op_data);
*
* H5D_chunk_iter_op_t parameters:
* hsize_t *offset; IN/OUT: Array of starting logical coordinates of chunk.
* uint32_t filter_mask; IN: Filter mask of chunk.
* haddr_t addr; IN: Offset in file of chunk data.
* uint32_t nbytes; IN: Size in number of bytes of chunk data in file.
* void *op_data; IN/OUT: Pointer to any user-defined data
* associated with the operation.
*
* The return values from an operator are:
* Zero (H5_ITER_CONT) causes the iterator to continue, returning zero when all
* elements have been processed.
* Positive (H5_ITER_STOP) causes the iterator to immediately return that positive
* value, indicating short-circuit success.
* Negative (H5_ITER_ERROR) causes the iterator to immediately return that value,
* indicating failure.
*
* Return: Non-negative on success, negative on failure
*
* Programmer: Gaute Hope
* August 2020
*
*-------------------------------------------------------------------------
*/
herr_t
H5Dchunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_chunk_iter_op_t op, void *op_data)
{
H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */
herr_t ret_value = SUCCEED;

FUNC_ENTER_API(FAIL)
H5TRACE4("e", "iix*x", dset_id, dxpl_id, op, op_data);

/* Check arguments */
if (NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier")
if (NULL == op)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid callback to chunk iteration")

/* Get the default dataset transfer property list if the user didn't provide one */
if (H5P_DEFAULT == dxpl_id)
dxpl_id = H5P_DATASET_XFER_DEFAULT;
else if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID")

/* Iterate over the chunks */
if ((ret_value = H5VL_dataset_optional(vol_obj, H5VL_NATIVE_DATASET_CHUNK_ITER, dxpl_id, H5_REQUEST_NULL,
op, op_data)) < 0)
HGOTO_ERROR(H5E_BADITER, H5E_BADITER, FAIL, "error iterating over dataset chunks")

done:
FUNC_LEAVE_API(ret_value)
} /* end H5Dchunk_iter() */
110 changes: 110 additions & 0 deletions src/H5Dchunk.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,12 @@ typedef struct H5D_chunk_coll_fill_info_t {
} H5D_chunk_coll_fill_info_t;
#endif /* H5_HAVE_PARALLEL */

typedef struct H5D_chunk_iter_ud_t {
H5D_chunk_iter_op_t op; /* User defined callback */
void *op_data; /* User data for user defined callback */
H5O_layout_chunk_t *chunk; /* Chunk layout */
} H5D_chunk_iter_ud_t;

/********************/
/* Local Prototypes */
/********************/
Expand All @@ -272,6 +278,7 @@ static herr_t H5D__chunk_dest(H5D_t *dset);
static int H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata);
static int H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata);
static int H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata);
static int H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata);

/* "Nonexistent" layout operation callback */
static ssize_t H5D__nonexistent_readvv(const H5D_io_info_t *io_info, size_t chunk_max_nseq,
Expand Down Expand Up @@ -7649,3 +7656,106 @@ H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned
done:
FUNC_LEAVE_NOAPI_TAG(ret_value)
} /* end H5D__get_chunk_info_by_coord() */

/*-------------------------------------------------------------------------
* Function: H5D__chunk_iter_cb
*
* Purpose: Call the user-defined function with the chunk data. The iterator continues if
* the user-defined function returns H5_ITER_CONT, and stops if H5_ITER_STOP is
* returned.
*
* Return: Success: H5_ITER_CONT or H5_ITER_STOP
* Failure: Negative (H5_ITER_ERROR)
*
* Programmer: Gaute Hope
* August 2020
*
*-------------------------------------------------------------------------
*/
static int
H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata)
{
const H5D_chunk_iter_ud_t *data = (H5D_chunk_iter_ud_t *)udata;
const H5O_layout_chunk_t *chunk = data->chunk;
int ret_value = H5_ITER_CONT;
hsize_t offset[H5O_LAYOUT_NDIMS];
unsigned ii; /* Match H5O_layout_chunk_t.ndims */

/* Similar to H5D__get_chunk_info */
for (ii = 0; ii < chunk->ndims; ii++)
offset[ii] = chunk_rec->scaled[ii] * chunk->dim[ii];

FUNC_ENTER_PACKAGE_NOERR

/* Check for callback failure and pass along return value */
if ((ret_value = (data->op)(offset, chunk_rec->filter_mask, chunk_rec->chunk_addr, chunk_rec->nbytes,
data->op_data)) < 0)
HERROR(H5E_DATASET, H5E_CANTNEXT, "iteration operator failed");

FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__chunk_iter_cb */

/*-------------------------------------------------------------------------
* Function: H5D__chunk_iter
*
* Purpose: Iterate over all the chunks in the dataset with given callback.
*
* Return: Success: Non-negative
* Failure: Negative
*
* Programmer: Gaute Hope
* August 2020
*
*-------------------------------------------------------------------------
*/
herr_t
H5D__chunk_iter(H5D_t *dset, H5D_chunk_iter_op_t op, void *op_data)
{
const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */
H5O_layout_t *layout = NULL; /* Dataset layout */
H5D_rdcc_ent_t *ent; /* Cache entry index */
H5D_chk_idx_info_t idx_info; /* Chunked index info */
herr_t ret_value = SUCCEED; /* Return value */

FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr)

/* Check args */
HDassert(dset);
HDassert(dset->shared);

/* Get dataset layout and raw data chunk cache */
layout = &(dset->shared->layout);
rdcc = &(dset->shared->cache.chunk);
HDassert(layout);
HDassert(rdcc);
HDassert(H5D_CHUNKED == layout->type);

/* Search for cached chunks that haven't been written out */
for (ent = rdcc->head; ent; ent = ent->next)
/* Flush the chunk out to disk, to make certain the size is correct later */
if (H5D__chunk_flush_entry(dset, ent, FALSE) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "cannot flush indexed storage buffer")

/* Compose chunked index info struct */
idx_info.f = dset->oloc.file;
idx_info.pline = &dset->shared->dcpl_cache.pline;
idx_info.layout = &layout->u.chunk;
idx_info.storage = &layout->storage.u.chunk;

/* If the dataset is not written, return without errors */
if (H5F_addr_defined(idx_info.storage->idx_addr)) {
H5D_chunk_iter_ud_t ud;

/* Set up info for iteration callback */
ud.op = op;
ud.op_data = op_data;
ud.chunk = &dset->shared->layout.u.chunk;

/* Iterate over the allocated chunks calling the iterator callback */
if ((ret_value = (layout->storage.u.chunk.ops->iterate)(&idx_info, H5D__chunk_iter_cb, &ud)) < 0)
HERROR(H5E_DATASET, H5E_CANTNEXT, "chunk iteration failed");
} /* end if H5F_addr_defined */

done:
FUNC_LEAVE_NOAPI_TAG(ret_value)
} /* end H5D__chunk_iter() */
1 change: 1 addition & 0 deletions src/H5Dpkg.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -565,6 +565,7 @@ H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_
unsigned *filter_mask, haddr_t *offset, hsize_t *size);
H5_DLL herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned *filter_mask,
haddr_t *addr, hsize_t *size);
H5_DLL herr_t H5D__chunk_iter(H5D_t *dset, H5D_chunk_iter_op_t cb, void *op_data);
H5_DLL haddr_t H5D__get_offset(const H5D_t *dset);
H5_DLL herr_t H5D__vlen_get_buf_size(H5D_t *dset, hid_t type_id, hid_t space_id, hsize_t *size);
H5_DLL herr_t H5D__vlen_get_buf_size_gen(H5VL_object_t *vol_obj, hid_t type_id, hid_t space_id,
Expand Down
60 changes: 54 additions & 6 deletions src/H5Dpublic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,27 @@ typedef herr_t (*H5D_scatter_func_t)(const void **src_buf /*out*/, size_t *src_b
typedef herr_t (*H5D_gather_func_t)(const void *dst_buf, size_t dst_buf_bytes_used, void *op_data);
//! <!-- [H5D_gather_func_t_snip] -->

//! <!-- [H5D_chunk_iter_op_t_snip] -->
/**
* \brief Callback for H5Dchunk_iter()
*
* \param[in] offset Logical position of the chunk’s first element in units of dataset elements
* \param[in] filter_mask Bitmask indicating the filters used when the chunk was written
* \param[in] addr Chunk address in the file
* \param[in] size Chunk size in bytes, 0 if the chunk does not exist
* \param[in,out] op_data Pointer to any user-defined data associated with
* the operation.
* \returns \li Zero (#H5_ITER_CONT) causes the iterator to continue, returning
* zero when all elements have been processed.
* \li A positive value (#H5_ITER_STOP) causes the iterator to
* immediately return that value, indicating short-circuit success.
* \li A negative (#H5_ITER_ERROR) causes the iterator to immediately
* return that value, indicating failure.
*/
typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t size,
void *op_data);
//! <!-- [H5D_chunk_iter_op_t_snip] -->

/********************/
/* Public Variables */
/********************/
Expand Down Expand Up @@ -588,10 +609,10 @@ H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks
* \brief Retrieves information about a chunk specified by its coordinates
*
* \dset_id
* \param[in] offset Logical position of the chunk’s first element
* \param[out] filter_mask Indicating filters used with the chunk when written
* \param[in] offset Logical position of the chunk’s first element in units of dataset elements
* \param[out] filter_mask Bitmask indicating the filters used when the chunk was written
* \param[out] addr Chunk address in the file
* \param[out] size Chunk size in bytes, 0 if chunk doesn’t exist
* \param[out] size Chunk size in bytes, 0 if the chunk does not exist
*
* \return \herr_t
*
Expand All @@ -612,6 +633,33 @@ H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks
*/
H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filter_mask,
haddr_t *addr, hsize_t *size);
/**
* --------------------------------------------------------------------------
* \ingroup H5D
*
* \brief Iterate over all chunks of a chunked dataset
*
* \dset_id
* \param[in] dxpl_id Identifier of a transfer property list
* \param[in] cb User callback function, called for every chunk.
* \param[in] op_data User-defined pointer to data required by op
*
* \return \herr_t
*
* \details H5Dchunk_iter iterates over all chunks in the dataset, calling the
* user supplied callback with the details of the chunk and the supplied
* context \p op_data.
*
* \par Example
* For each chunk, print the allocated chunk size (0 for un-allocated chunks).
* \snippet H5D_examples.c H5Dchunk_iter_cb
* Iterate over all chunked datasets and chunks in a file.
* \snippet H5D_examples.c H5Ovisit_cb
*
* \since 1.12.3, 1.13.0
*
*/
H5_DLL herr_t H5Dchunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_chunk_iter_op_t cb, void *op_data);

/**
* --------------------------------------------------------------------------
Expand All @@ -622,10 +670,10 @@ H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, u
* \dset_id
* \param[in] fspace_id File dataspace selection identifier (See Note below)
* \param[in] chk_idx Index of the chunk
* \param[out] offset Logical position of the chunk’s first element
* \param[out] filter_mask Indicating filters used with the chunk when written
* \param[out] offset Logical position of the chunk’s first element in units of dataset elements
* \param[out] filter_mask Bitmask indicating the filters used when the chunk was written
* \param[out] addr Chunk address in the file
* \param[out] size Chunk size in bytes, 0 if chunk doesn’t exist
* \param[out] size Chunk size in bytes, 0 if the chunk does not exist
*
* \return \herr_t
*
Expand Down
21 changes: 11 additions & 10 deletions src/H5VLnative.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,17 @@
#endif /* H5_NO_DEPRECATED_SYMBOLS */

/* Values for native VOL connector dataset optional VOL operations */
#define H5VL_NATIVE_DATASET_FORMAT_CONVERT 0 /* H5Dformat_convert (internal) */
#define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE 1 /* H5Dget_chunk_index_type */
#define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE 2 /* H5Dget_chunk_storage_size */
#define H5VL_NATIVE_DATASET_GET_NUM_CHUNKS 3 /* H5Dget_num_chunks */
#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX 4 /* H5Dget_chunk_info */
#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD 5 /* H5Dget_chunk_info_by_coord */
#define H5VL_NATIVE_DATASET_CHUNK_READ 6 /* H5Dchunk_read */
#define H5VL_NATIVE_DATASET_CHUNK_WRITE 7 /* H5Dchunk_write */
#define H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE 8 /* H5Dvlen_get_buf_size */
#define H5VL_NATIVE_DATASET_GET_OFFSET 9 /* H5Dget_offset */
#define H5VL_NATIVE_DATASET_FORMAT_CONVERT 0 /* H5Dformat_convert (internal) */
#define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE 1 /* H5Dget_chunk_index_type */
#define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE 2 /* H5Dget_chunk_storage_size */
#define H5VL_NATIVE_DATASET_GET_NUM_CHUNKS 3 /* H5Dget_num_chunks */
#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX 4 /* H5Dget_chunk_info */
#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD 5 /* H5Dget_chunk_info_by_coord */
#define H5VL_NATIVE_DATASET_CHUNK_READ 6 /* H5Dchunk_read */
#define H5VL_NATIVE_DATASET_CHUNK_WRITE 7 /* H5Dchunk_write */
#define H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE 8 /* H5Dvlen_get_buf_size */
#define H5VL_NATIVE_DATASET_GET_OFFSET 9 /* H5Dget_offset */
#define H5VL_NATIVE_DATASET_CHUNK_ITER 10 /* H5Dchunk_iter */

/* Values for native VOL connector file optional VOL operations */
#define H5VL_NATIVE_FILE_CLEAR_ELINK_CACHE 0 /* H5Fclear_elink_file_cache */
Expand Down
19 changes: 19 additions & 0 deletions src/H5VLnative_dataset.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -587,6 +587,25 @@ H5VL__native_dataset_optional(void *obj, H5VL_dataset_optional_t optional_type,
break;
}

/* H5Dchunk_iter */
case H5VL_NATIVE_DATASET_CHUNK_ITER: {
H5D_chunk_iter_op_t op = HDva_arg(arguments, H5D_chunk_iter_op_t);
void *op_data = HDva_arg(arguments, void *);

/* Sanity check */
HDassert(dset->shared);

/* Make sure the dataset is chunked */
if (H5D_CHUNKED != dset->shared->layout.type)
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset")

/* Call private function */
if ((ret_value = H5D__chunk_iter(dset, op, op_data)) < 0)
HERROR(H5E_DATASET, H5E_BADITER, "chunk iteration failed");

break;
}

default:
HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "invalid optional operation")
} /* end switch */
Expand Down
Loading

0 comments on commit 0c1898d

Please sign in to comment.