Skip to content

Commit

Permalink
Subfiling updates for release (HDFGroup#1963)
Browse files Browse the repository at this point in the history 
* Remove generated file h5fuse.sh

* Link pthreads library when Subfiling VFD is built

* Switch to MPI I/O driver for Subfiling HDF5 stub file

* Rough first implementation for Subfiling file deletion

* Subfiling VFD - get file dirname for file deletion

* Subfiling VFD - set lock callback to NULL for now to avoid performance
issues

* Committing clang-format changes

* Minor tidying up of Subfiling testing

* Fixups for Subfiling VFD support in tools

* Tidy up Subfiling public interface and add Doxygen

* Respect Subfiling configuration settings from application

* Add release note for Subfiling VFD

* Committing clang-format changes

* Committing clang-format changes

* Shorten some Subfiling environment variable names

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
  • Loading branch information
@jhendersonHDF @github-actions
jhendersonHDF and github-actions[bot] authored Aug 4, 2022
1 parent a71534f commit bf07e0f
Show file tree
Hide file tree
Showing 16 changed files with 1,004 additions and 574 deletions.
2 changes: 1 addition & 1 deletion bin/trace
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ $Source = "";
"H5_index_t" => "Ii",
"H5I_iterate_func_t" => "II",
"H5_iter_order_t" => "Io",
"ioc_selection_t" => "IO",
"H5FD_subfiling_ioc_select_t" => "IO",
"H5I_future_realize_func_t" => "IR",
"int" => "Is",
"int32_t" => "Is",
Expand Down
2 changes: 1 addition & 1 deletion doxygen/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ if (DOXYGEN_FOUND)
set (DOXYGEN_SEARCHENGINE_URL)
set (DOXYGEN_STRIP_FROM_PATH ${HDF5_SOURCE_DIR})
set (DOXYGEN_STRIP_FROM_INC_PATH ${HDF5_SOURCE_DIR})
set (DOXYGEN_PREDEFINED "H5_HAVE_DIRECT H5_HAVE_LIBHDFS H5_HAVE_MAP_API H5_HAVE_PARALLEL H5_HAVE_ROS3_VFD")
set (DOXYGEN_PREDEFINED "H5_HAVE_DIRECT H5_HAVE_LIBHDFS H5_HAVE_MAP_API H5_HAVE_PARALLEL H5_HAVE_ROS3_VFD H5_HAVE_SUBFILING_VFD H5_HAVE_IOC_VFD")

# This configure and individual custom targets work together
# Replace variables inside @@ with the current values
Expand Down
2 changes: 2 additions & 0 deletions doxygen/Doxyfile.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -860,6 +860,7 @@ FILE_PATTERNS = H5*public.h \
H5FDdirect.h \
H5FDfamily.h \
H5FDhdfs.h \
H5FDioc.h \
H5FDlog.h \
H5FDmirror.h \
H5FDmpi.h \
Expand All @@ -869,6 +870,7 @@ FILE_PATTERNS = H5*public.h \
H5FDsec2.h \
H5FDsplitter.h \
H5FDstdio.h \
H5FDsubfiling.h \
H5FDwindows.h \
H5VLconnector.h \
H5VLconnector_passthru.h \
Expand Down
23 changes: 22 additions & 1 deletion release_docs/RELEASE.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,28 @@ New Features

Library:
--------
-
- Subfiling VFD

The HDF5 Subfiling VFD is a new MPI-based file driver that allows an
HDF5 application to distribute an HDF5 file across a collection of
"sub-files" in equal-sized data segment "stripes". I/O to the logical
HDF5 file is then directed to the appropriate "sub-file" according to
the Subfiling configuration and a system of I/O concentrators, which
are MPI ranks operating worker threads.

By allowing a configurable stripe size, number of I/O concentrators and
method for selecting MPI ranks as I/O concentrators, the Subfiling VFD
aims to enable an HDF5 application to find a middle ground between the
single shared file and file-per-process approaches to parallel file I/O
for the particular machine the application is running on. In general, the
goal is to avoid some of the complexity of the file-per-process approach
while also minimizing the locking issues of the single shared file approach
on a parallel file system.

The Subfiling VFD can be used by calling H5Pset_fapl_subfiling() on a
File Access Property List and using that FAPL for file operations.

(JTH - 2022/07/22)


Parallel Library:
Expand Down
277 changes: 200 additions & 77 deletions src/H5FDsubfiling/H5FDioc.c
View file

Large diffs are not rendered by default.

193 changes: 156 additions & 37 deletions src/H5FDsubfiling/H5FDioc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

/*
* Purpose: The public header file for the "io concentrator" driver.
* Purpose: The public header file for the "I/O concentrator" driver.
* This provides a similar functionality to that of the subfiling driver
* but introduces the necessary file access functionality via a multi-
* threading MPI service
Expand All @@ -20,72 +20,191 @@
#ifndef H5FDioc_H
#define H5FDioc_H

#include "H5FDsubfiling.h"

#ifdef H5_HAVE_IOC_VFD
/**
* \def H5FD_IOC
* Macro that returns the identifier for the #H5FD_IOC driver. \hid_t{file driver}
*/
#define H5FD_IOC (H5FDperform_init(H5FD_ioc_init))
#else
#define H5FD_IOC (H5I_INVALID_HID)
#endif

/**
* \def H5FD_IOC_NAME
* The canonical name for the #H5FD_IOC driver
*/
#define H5FD_IOC_NAME "ioc"

#ifdef H5_HAVE_IOC_VFD

#ifndef H5FD_IOC_FAPL_MAGIC
#define H5FD_CURR_IOC_FAPL_VERSION 1
#define H5FD_IOC_FAPL_MAGIC 0xFED21331
/**
* \def H5FD_IOC_CURR_FAPL_VERSION
* The version number of the H5FD_ioc_config_t configuration
* structure for the #H5FD_IOC driver
*/
#define H5FD_IOC_CURR_FAPL_VERSION 1
/**
* \def H5FD_IOC_FAPL_MAGIC
* Unique number used to distinguish the #H5FD_IOC driver from other HDF5 file drivers
*/
#define H5FD_IOC_FAPL_MAGIC 0xFED21331
#endif

#define H5FD_IOC_THREAD_POOL_SIZE 4
/**
* \def H5FD_IOC_DEFAULT_THREAD_POOL_SIZE
* The default number of I/O concentrator worker threads
*/
#define H5FD_IOC_DEFAULT_THREAD_POOL_SIZE 4

/*
* Environment variables interpreted by the IOC VFD
*/
#define H5_IOC_THREAD_POOL_COUNT "H5_IOC_THREAD_POOL_COUNT"

/*
* Define the various constants to allow different allocations
* of subfile ranks. The choices are self explanatory, starting
* with the default of one IO Concentrator (IOC) per node and
* lastly, defining a fixed number.
/**
* \def H5FD_IOC_THREAD_POOL_SIZE
* Macro for name of the environment variable that controls/overrides
* the number of I/O concentrator worker threads
*
* The value set for this environment variable is interpreted as an
* int value and must be > 0.
*/
typedef enum {
SELECT_IOC_ONE_PER_NODE = 0, /* Default */
SELECT_IOC_EVERY_NTH_RANK, /* Starting at rank 0, select-next += N */
SELECT_IOC_WITH_CONFIG, /* NOT IMPLEMENTED: Read-from-file */
SELECT_IOC_TOTAL, /* Starting at rank 0, mpi_size / total */
ioc_selection_options /* (Uses same selection as every Nth rank) */
} ioc_selection_t;
#define H5FD_IOC_THREAD_POOL_SIZE "H5FD_IOC_THREAD_POOL_SIZE"

/*
* In addition to the common configuration fields, we can have
* VFD specific fields. Here's one for the IO Concentrator VFD.
//! <!-- [H5FD_ioc_config_t_snip] -->
/**
* \struct H5FD_ioc_config_t
* \brief Configuration structure for H5Pset_fapl_ioc() / H5Pget_fapl_ioc()
*
* thread_pool_count (int32_t)
* Indicate the number of helper threads that we want for
* creating a thread pool
* \details H5FD_ioc_config_t is a public structure that is used to pass
* configuration data to the #H5FD_IOC driver via a File Access
* Property List. A pointer to an instance of this structure is
* a parameter to H5Pset_fapl_ioc() and H5Pget_fapl_ioc().
*
* The #H5FD_IOC driver shares much of its configuration with the
* #H5FD_SUBFILING driver and so its configuration structure
* contains an instance of a H5FD_subfiling_shared_config_t
* configuration structure.
*
* \var uint32_t H5FD_ioc_config_t::magic
* A somewhat unique number which distinguishes the #H5FD_IOC driver
* from other drivers. Used in combination with a version number, it
* can help to validate a user-generated File Access Property List.
* This field should be set to #H5FD_IOC_FAPL_MAGIC.
*
* \var uint32_t H5FD_ioc_config_t::version
* Version number of the H5FD_ioc_config_t structure. Any instance passed
* to H5Pset_fapl_ioc() / H5Pget_fapl_ioc() must have a recognized version
* number or an error will be raised. Currently, this field should be set
* to #H5FD_IOC_CURR_FAPL_VERSION.
*
* \var hid_t H5FD_ioc_config_t::under_fapl_id
* The File Access Property List which is setup with the file driver
* to use for I/O to the HDF5 stub file. The stub file looks like a
* typical HDF5 file, but currently only contains the superblock metadata
* for compatibility with legacy HDF5 applications. The default driver used
* is currently the #H5FD_MPIO driver.
*
* \var int32_t H5FD_ioc_config_t::thread_pool_count
* The number of I/O concentrator worker threads to use.
*
* This value can also be set or adjusted with the #H5FD_IOC_THREAD_POOL_SIZE
* environment variable.
*
* \var H5FD_subfiling_shared_config_t H5FD_ioc_config_t::subf_config
* Subfiling configuration data for the parent #H5FD_SUBFILING driver. This
* includes the sub-file stripe size, number of I/O concentrators, IOC
* selection method, etc.
*
* ----------------------------------------------------------------------------
*/

typedef struct H5FD_ioc_config_t {
uint32_t magic; /* set to H5FD_IOC_FAPL_MAGIC */
uint32_t version; /* set to H5FD_CURR_IOC_FAPL_VERSION */
int32_t stripe_count; /* How many io concentrators */
int64_t stripe_depth; /* Max # of bytes in contiguous IO to an IOC */
ioc_selection_t ioc_selection; /* Method to select IO Concentrators */
hid_t ioc_fapl_id; /* The hid_t value of the stacked VFD */
int32_t thread_pool_count;
uint32_t magic; /* Must be set to H5FD_IOC_FAPL_MAGIC */
uint32_t version; /* Must be set to H5FD_IOC_CURR_FAPL_VERSION */
hid_t under_fapl_id; /* FAPL setup with the VFD to use for I/O to the HDF5 stub file */
int32_t thread_pool_count; /* Number of I/O concentrator worker threads to use */
H5FD_subfiling_shared_config_t subf_config; /* Subfiling driver configuration */
} H5FD_ioc_config_t;
//! <!-- [H5FD_ioc_config_t_snip] -->

#ifdef __cplusplus
extern "C" {
#endif

H5_DLL hid_t H5FD_ioc_init(void);
H5_DLL herr_t H5Pset_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *config_ptr);
H5_DLL herr_t H5Pget_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *config_ptr);
H5_DLL void begin_thread_exclusive(void);
H5_DLL void end_thread_exclusive(void);
/**
* \brief Internal routine to initialize #H5FD_IOC driver. Not meant to be
* called directly by an HDF5 application
*/
H5_DLL hid_t H5FD_ioc_init(void);
/**
* \ingroup FAPL
*
* \brief Modifies the specified File Access Property List to use the #H5FD_IOC driver
*
* \fapl_id
* \param[in] vfd_config Pointer to #H5FD_IOC driver configuration structure. May be NULL.
* \returns \herr_t
*
* \details H5Pset_fapl_ioc() modifies the File Access Property List to use the
* #H5FD_IOC driver.
*
* The #H5FD_IOC driver is a reference implementation of an "I/O concentrator"
* file driver that works in conjunction with the #H5FD_SUBFILING driver and
* provides the I/O backend for servicing I/O requests to sub-files.
*
* Typically, an HDF5 application won't need to call this routine directly.
* The #H5FD_IOC driver is usually set up as a side effect of an HDF5 application
* using the #H5FD_SUBFILING driver, but this routine is provided in case the
* application wishes to manually configure the #H5FD_IOC driver.
*
* \note The \p vfd_config parameter may be NULL. In this case, the driver will
* be setup with default settings. Note that in this case, it is assumed
* the parent #H5FD_SUBFILING driver was also setup with default settings.
* If the two drivers differ in configuration settings, application behavior
* may not be as expected.
*
* \since 1.13.2
*
*/
H5_DLL herr_t H5Pset_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *vfd_config);
/**
* \ingroup FAPL
*
* \brief Queries a File Access Property List for #H5FD_IOC file driver properties
*
* \fapl_id
* \param[out] config_out Pointer to H5FD_ioc_config_t structure through which the
* #H5FD_IOC file driver properties will be returned.
*
* \returns \herr_t
*
* \details H5Pget_fapl_ioc() queries the specified File Access Property List for
* #H5FD_IOC driver properties as set by H5Pset_fapl_ioc(). If the #H5FD_IOC
* driver has not been set on the File Access Property List, a default
* configuration is returned. An HDF5 application may use this functionality
* to manually configure the #H5FD_IOC driver by calling H5Pget_fapl_ioc()
* on a newly-created File Access Property List, adjusting the default
* values and then calling H5Pset_fapl_ioc() with the configured
* H5FD_ioc_config_t structure.
*
* \since 1.13.2
*
*/
H5_DLL herr_t H5Pget_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *config_out);
/**
* \brief Internal routine for managing exclusive access to critical sections
* by the #H5FD_IOC driver's worker threads. Not meant to be called
* directly by an HDF5 application
*/
H5_DLL void H5FD_ioc_begin_thread_exclusive(void);
/**
* \brief Internal routine for managing exclusive access to critical sections
* by the #H5FD_IOC driver's worker threads. Not meant to be called
* directly by an HDF5 application
*/
H5_DLL void H5FD_ioc_end_thread_exclusive(void);

#ifdef __cplusplus
}
Expand Down
20 changes: 8 additions & 12 deletions src/H5FDsubfiling/H5FDioc_threads.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,6 @@

#include "H5FDsubfiling.h"

#ifndef HG_TEST_NUM_THREADS_DEFAULT
#define HG_TEST_NUM_THREADS_DEFAULT 4
#endif

#define MIN_READ_RETRIES 10

/*
Expand Down Expand Up @@ -118,7 +114,7 @@ initialize_ioc_threads(void *_sf_context)
{
subfiling_context_t *sf_context = _sf_context;
ioc_data_t *ioc_data = NULL;
unsigned thread_pool_count = HG_TEST_NUM_THREADS_DEFAULT;
unsigned thread_pool_count = H5FD_IOC_DEFAULT_THREAD_POOL_SIZE;
char *env_value;
int ret_value = 0;
#ifdef H5FD_IOC_COLLECT_STATS
Expand Down Expand Up @@ -174,7 +170,7 @@ initialize_ioc_threads(void *_sf_context)
H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, (-1), "can't initialize IOC thread queue mutex");

/* Allow experimentation with the number of helper threads */
if ((env_value = HDgetenv(H5_IOC_THREAD_POOL_COUNT)) != NULL) {
if ((env_value = HDgetenv(H5FD_IOC_THREAD_POOL_SIZE)) != NULL) {
int value_check = HDatoi(env_value);
if (value_check > 0) {
thread_pool_count = (unsigned int)value_check;
Expand Down Expand Up @@ -589,7 +585,7 @@ handle_work_request(void *arg)
}

/*-------------------------------------------------------------------------
* Function: begin_thread_exclusive
* Function: H5FD_ioc_begin_thread_exclusive
*
* Purpose: Mutex lock to restrict access to code or variables.
*
Expand All @@ -603,13 +599,13 @@ handle_work_request(void *arg)
*-------------------------------------------------------------------------
*/
void
begin_thread_exclusive(void)
H5FD_ioc_begin_thread_exclusive(void)
{
hg_thread_mutex_lock(&ioc_thread_mutex);
}

/*-------------------------------------------------------------------------
* Function: end_thread_exclusive
* Function: H5FD_ioc_end_thread_exclusive
*
* Purpose: Mutex unlock. Should only be called by the current holder
* of the locked mutex.
Expand All @@ -624,7 +620,7 @@ begin_thread_exclusive(void)
*-------------------------------------------------------------------------
*/
void
end_thread_exclusive(void)
H5FD_ioc_end_thread_exclusive(void)
{
hg_thread_mutex_unlock(&ioc_thread_mutex);
}
Expand Down Expand Up @@ -840,13 +836,13 @@ ioc_file_queue_write_indep(sf_work_request_t *msg, int subfile_rank, int source,
sf_queue_delay_time += t_queue_delay;
#endif

begin_thread_exclusive();
H5FD_ioc_begin_thread_exclusive();

/* Adjust EOF if necessary */
if (sf_eof > sf_context->sf_eof)
sf_context->sf_eof = sf_eof;

end_thread_exclusive();
H5FD_ioc_end_thread_exclusive();

done:
if (send_nack) {
Expand Down
Loading

0 comments on commit bf07e0f

Please sign in to comment.