Merge branch 'master' of https://github.com/h5py/h5py into ros3-token

# Conflicts:
#	h5py/api_functions.txt

.github/ISSUE_TEMPLATE.md

@@ -1,10 +1,10 @@
 To assist reproducing bugs, please include the following:
- * Operating System (e.g. Windows 10, MacOS 10.11, Ubuntu 16.04.2 LTS, CentOS 7)
- * Python version (e.g. 2.7, 3.5)
+ * Operating System (e.g. Windows 10, MacOS 10.11, Ubuntu 20.04.2 LTS, CentOS 7)
+ * Python version (e.g. 3.10)
  * Where Python was acquired (e.g. system Python on MacOS or Linux, Anaconda on
    Windows)
- * h5py version (e.g. 2.6)
- * HDF5 version (e.g. 1.8.17)
+ * h5py version (e.g. 3.9)
+ * HDF5 version (e.g. 1.12.2)
  * The full traceback/stack trace shown (if it appears)
 
 `h5py.version.info` contains the needed versions, which can be displayed by

.travis.yml

@@ -54,7 +54,7 @@ before_install:
 install:
     - pip install -U pip tox codecov virtualenv
     - ci/get_hdf5_if_needed.sh
-    - if [[ $CIBW_ARCHS == aarch64 ]]; then python -m pip install cibuildwheel; fi
+    - if [[ $CIBW_ARCHS == aarch64 ]]; then python -m pip install 'cibuildwheel<2.15'; fi
 
 script:
     - if [[ $CIBW_ARCHS == aarch64 ]]; then python3 -m cibuildwheel --output-dir wheelhouse; fi

README.rst

@@ -8,7 +8,7 @@
 HDF5 for Python
 ===============
 `h5py` is a thin, pythonic wrapper around `HDF5 <https://portal.hdfgroup.org/display/HDF5/>`_,
-which runs on Python 3 (3.7+).
+which runs on Python 3 (3.8+).
 
 Websites
 --------

api_gen.py

@@ -47,13 +47,13 @@ class Line:
         sig:        String with raw function signature
         args:       String with sequence of arguments to call function
 
-        Example:    MPI 1.8.12 int foo(char* a, size_t b)
+        Example:    MPI 1.12.2 int foo(char* a, size_t b)
 
         .nogil:     ""
         .mpi:       True
         .ros3:      False
         .direct_vfd: False
-        .version:   (1, 8, 12)
+        .version:   (1, 12, 2)
         .code:      "int"
         .fname:     "foo"
         .sig:       "char* a, size_t b"

appveyor.yml

@@ -1,19 +1,11 @@
 # from https://packaging.python.org/en/latest/appveyor/
 image: Visual Studio 2019
 environment:
-  HDF5_VERSION: "1.8.17"
   TOX_TESTENV_PASSENV: "HDF5_DIR"
   H5PY_ENFORCE_COVERAGE: "YES"
 
   matrix:
 
-    ### USING HDF5 1.8 ###
-
-    - PYTHON: "C:\\Python38"
-      TOXENV: "py38-test-deps"
-      HDF5_VSVERSION: "14"
-      HDF5_DIR: "C:\\hdf5\\%HDF5_VERSION%\\%HDF5_VSVERSION%"
-
     ### USING HDF5 1.10 ###
 
     - PYTHON: "C:\\Python38"

azure-pipelines.yml

@@ -27,7 +27,7 @@ jobs:
   pool:
     vmImage: ubuntu-20.04
   variables:
-    CIBW_BUILD: cp3{8,9,10,11}-manylinux_x86_64
+    CIBW_BUILD: cp3{8,9,10,11,12}-manylinux_x86_64
     CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/h5py/manylinux2014_x86_64-hdf5
     # Include less debugging info for smaller wheels (default is -g2)
     CIBW_ENVIRONMENT: "CFLAGS=-g1"
@@ -40,10 +40,10 @@ jobs:
   pool:
     vmImage: windows-2019
   variables:
-    HDF5_VERSION: 1.12.2
+    HDF5_VERSION: 1.14.2
     HDF5_DIR: $(HDF5_CACHE_DIR)/$(HDF5_VERSION)
     HDF5_VSVERSION: "16-64"
-    CIBW_BUILD: cp3{8,9,10,11}-win_amd64
+    CIBW_BUILD: cp3{8,9,10,11,12}-win_amd64
   steps:
     - template: ci/azure-pipelines-wheels.yml
       parameters:
@@ -60,7 +60,7 @@ jobs:
     HDF5_DIR: $(HDF5_CACHE_DIR)/$(HDF5_VERSION)
     CIBW_BUILD_VERBOSITY_MACOS: 3
     CIBW_ARCHS_MACOS: $(arch)
-    CIBW_BUILD: cp3{8,9,10,11}-macosx_*
+    CIBW_BUILD: cp3{8,9,10,11,12}-macosx_*
     CIBW_ENVIRONMENT_MACOS: LD_LIBRARY_PATH="$(HDF5_CACHE_DIR)/$(HDF5_VERSION)/lib:${LD_LIBRARY_PATH}" PKG_CONFIG_PATH="$(HDF5_CACHE_DIR)/$(HDF5_VERSION)/lib/pkgconfig:${PKG_CONFIG_PATH}"
     CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
       DYLD_FALLBACK_LIBRARY_PATH=$(HDF5_CACHE_DIR)/$(HDF5_VERSION)/lib delocate-listdeps {wheel} &&
@@ -113,10 +113,10 @@ jobs:
         HDF5_VERSION: 1.12.2
         HDF5_DIR: $(HDF5_CACHE_DIR)/$(HDF5_VERSION)
         H5PY_ENFORCE_COVERAGE: yes
-      py310-deps-hdf51140:
+      py310-deps-hdf51142:
         python.version: '3.10'
         TOXENV: py310-test-deps
-        HDF5_VERSION: 1.14.0
+        HDF5_VERSION: 1.14.2
         HDF5_DIR: $(HDF5_CACHE_DIR)/$(HDF5_VERSION)
         H5PY_ENFORCE_COVERAGE: yes
     # do mpi tests
@@ -163,6 +163,10 @@ jobs:
         python.version: '3.11'
         TOXENV: py311-test-deps,py311-test-mindeps,py311-test-deps-pre
         WHL_FILE: h5py*-cp311-*manylinux2014_x86_64.whl
+      py312:
+        python.version: '3.12'
+        TOXENV: py312-test-deps,py312-test-mindeps,py312-test-deps-pre
+        WHL_FILE: h5py*-cp312-*manylinux2014_x86_64.whl
 
   steps:
     - template: ci/azure-pipelines-test-wheels.yml
@@ -179,14 +183,6 @@ jobs:
     # -mindeps : test with oldest supported version of (python) dependencies
     # -deps-pre : test pre-release versions of (python) dependencies)
     #
-    # 64 bit - HDF5 1.8
-      py38-hdf518:
-        python.version: '3.8'
-        TOXENV: py38-test-deps
-        HDF5_VERSION: 1.8.17
-        HDF5_DIR: $(HDF5_CACHE_DIR)/$(HDF5_VERSION)
-        HDF5_VSVERSION: "16-64"
-        H5PY_ENFORCE_COVERAGE: yes
     # 64 bit - HDF5 1.10
       py39-hdf5110:
         python.version: '3.9'
@@ -204,10 +200,11 @@ jobs:
         HDF5_DIR: $(HDF5_CACHE_DIR)/$(HDF5_VERSION)
         HDF5_VSVERSION: "16-64"
         H5PY_ENFORCE_COVERAGE: yes
+    # 64 bit - HDF5 1.14
       py310-hdf5114:
         python.version: '3.10'
         TOXENV: py310-test-deps
-        HDF5_VERSION: 1.14.0
+        HDF5_VERSION: 1.14.2
         HDF5_DIR: $(HDF5_CACHE_DIR)/$(HDF5_VERSION)
         HDF5_VSVERSION: "16-64"
         H5PY_ENFORCE_COVERAGE: yes
@@ -245,6 +242,10 @@ jobs:
         python.version: '3.11'
         TOXENV: py311-test-deps,py311-test-mindeps,py311-test-deps-pre
         WHL_FILE: h5py*-cp311-*.whl
+      py312:
+        python.version: '3.12'
+        TOXENV: py312-test-deps,py312-test-mindeps,py312-test-deps-pre
+        WHL_FILE: h5py*-cp312-*.whl
 
   steps:
     - template: ci/azure-pipelines-test-wheels.yml
@@ -301,6 +302,10 @@ jobs:
         python.version: '3.11'
         TOXENV: py311-test-deps,py311-test-mindeps,py311-test-deps-pre
         WHL_FILE: h5py*-cp311-*.whl
+      py312:
+        python.version: '3.12'
+        TOXENV: py312-test-deps,py312-test-mindeps,py312-test-deps-pre
+        WHL_FILE: h5py*-cp312-*.whl
 
   steps:
     - template: ci/azure-pipelines-test-wheels.yml

ci/azure-pipelines-wheels.yml

@@ -34,7 +34,7 @@ steps:
 - bash: |
     set -o errexit
     python -m pip install --upgrade pip
-    pip install cibuildwheel==2.11.1
+    pip install cibuildwheel==2.16.2
   displayName: 'Install cibuildwheel'
 
 - script: env

ci/fix_paths.py

@@ -1,6 +1,6 @@
-import distutils.sysconfig
-from glob import glob
 import os
+import sysconfig
+from glob import glob
 from os.path import join as pjoin, basename
 from shutil import copy
 from sys import platform
@@ -9,8 +9,7 @@ def main():
     """
     Copy HDF5 DLLs into installed h5py package
     """
-    # This is the function Tox also uses to locate site-packages (Apr 2019)
-    sitepackagesdir = distutils.sysconfig.get_python_lib(plat_specific=True)
+    sitepackagesdir = sysconfig.get_path('platlib')
     print("site packages dir:", sitepackagesdir)
 
     hdf5_path = os.environ.get("HDF5_DIR")

ci/get_hdf5_win.py

@@ -37,7 +37,7 @@
 CMAKE_INSTALL_PATH_ARG = "-DCMAKE_INSTALL_PREFIX={install_path}"
 CMAKE_HDF5_LIBRARY_PREFIX = ["-DHDF5_EXTERNAL_LIB_PREFIX=h5py_"]
 REL_PATH_TO_CMAKE_CFG = "hdf5-{version}"
-DEFAULT_VERSION = '1.8.17'
+DEFAULT_VERSION = '1.12.2'
 VSVERSION_TO_GENERATOR = {
     "9": "Visual Studio 9 2008",
     "10": "Visual Studio 10 2010",

docs/build.rst

@@ -78,11 +78,13 @@ Source installation
 To install h5py from source, you need:
 
 * A supported Python version with development headers
-* HDF5 1.8.4 or newer with development headers
+* HDF5 1.10.4 or newer with development headers
 
   * HDF5 versions newer than the h5py version you're using might not work.
   * Odd minor versions of HDF5 (e.g. 1.13) are experimental, and might not work.
     Use a 'maintenance' version like 1.12.x if possible.
+  * If you need support for older HDF5 versions, h5py up to version 3.9
+    supported HDF5 1.8.4 and above.
 
 * A C compiler
 

docs/contributing.rst

@@ -335,8 +335,8 @@ If the tests pass, the feature is ready for a pull request.
 Adding a function only available in certain versions of HDF5
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-At the moment, h5py must be backwards-compatible all the way back to
-HDF5 1.8.4.  Starting with h5py 2.2.0, it's possible to conditionally
+At the moment, h5py must be compatible with HDF5 back to version 1.10.4.
+But it's possible to conditionally
 include functions which only appear in newer versions of HDF5.  It's also
 possible to mark functions which require Parallel HDF5.  For example, the
 function ``H5Fset_mpi_atomicity`` was introduced in HDF5 1.8.9 and requires

docs/high/group.rst

@@ -135,7 +135,7 @@ If the target is removed, they will "dangle":
 External links
 ~~~~~~~~~~~~~~
 
-New in HDF5 1.8, external links are "soft links plus", which allow you to
+External links are "soft links plus", which allow you to
 specify the name of the file as well as the path to the desired object.  You
 can refer to objects in any file you wish.  Use similar syntax as for soft
 links:

docs/mpi.rst

@@ -138,7 +138,7 @@ On the other hand, writing data to a dataset can be done independently::
 MPI atomic mode
 ---------------
 
-HDF5 versions 1.8.9+ support the MPI "atomic" file access mode, which trades
+HDF5 supports the MPI "atomic" file access mode, which trades
 speed for more stringent consistency requirements.  Once you've opened a
 file with the ``mpio`` driver, you can place it in atomic mode using the
 settable ``atomic`` property::

h5py/__init__.py

@@ -67,6 +67,7 @@
 from ._hl.dataset import Dataset
 from ._hl.datatype import Datatype
 from ._hl.attrs import AttributeManager
+from ._hl.vds import VirtualSource, VirtualLayout
 
 from ._selector import MultiBlockSlice
 from .h5 import get_config
@@ -82,10 +83,6 @@
 from .version import version as __version__
 
 
-if version.hdf5_version_tuple[:3] >= get_config().vds_min_hdf5_version:
-    from ._hl.vds import VirtualSource, VirtualLayout
-
-
 def run_tests(args=''):
     """Run tests with pytest and returns the exit status as an int.
     """

h5py/_conv.pyx

@@ -22,11 +22,12 @@ from .h5t cimport H5PY_OBJ, typewrap, py_create, TypeID, H5PY_PYTHON_OPAQUE_TAG
 from libc.stdlib cimport realloc
 from libc.string cimport strcmp
 from .utils cimport emalloc, efree
+from ._proxy cimport needs_bkg_buffer
 cfg = get_config()
 
 # Initialization of numpy
 cimport numpy as cnp
-from numpy cimport npy_intp, NPY_WRITEABLE, NPY_C_CONTIGUOUS, NPY_OWNDATA
+from numpy cimport npy_intp, NPY_WRITEABLE, NPY_C_CONTIGUOUS, NPY_OWNDATA, PyArray_DATA
 cnp._import_array()
 import numpy as np
 
@@ -158,10 +159,7 @@ cdef bint _is_pyobject_opaque(hid_t obj):
                     return True
         return False
     finally:
-        IF HDF5_VERSION >= (1, 8, 13):
-            H5free_memory(ctag)
-        ELSE:
-            free(ctag)
+        H5free_memory(ctag)
 
 cdef herr_t init_vlen2str(hid_t src_vlen, hid_t dst_str, void** priv) except -1:
     # /!\ Untested
@@ -700,7 +698,8 @@ cdef int conv_vlen2ndarray(void* ipt,
         int flags = NPY_WRITEABLE | NPY_C_CONTIGUOUS | NPY_OWNDATA
         npy_intp dims[1]
         void* data
-        cdef char[:] buf
+        char[:] buf
+        void* back_buf = NULL
         cnp.ndarray ndarray
         PyObject* ndarray_obj
         vlen_t in_vlen0
@@ -714,17 +713,42 @@ cdef int conv_vlen2ndarray(void* ipt,
     itemsize = H5Tget_size(outtype.id)
     if itemsize > H5Tget_size(intype.id):
         data = realloc(data, itemsize * size)
-    H5Tconvert(intype.id, outtype.id, size, data, NULL, H5P_DEFAULT)
+
+    if needs_bkg_buffer(intype.id, outtype.id):
+        back_buf = emalloc(H5Tget_size(outtype.id)*size)
+
+    try:
+        H5Tconvert(intype.id, outtype.id, size, data, back_buf, H5P_DEFAULT)
+    finally:
+        free(back_buf)
+
+    # We need to use different approaches to creating the ndarray with the converted
+    # data depending on the destination dtype.
+    # For simple dtypes, we can use SimpleNewFromData, but types like
+    # string & void need a size specified, so this function can't be used.
+    # Additionally, Cython doesn't expose NumPy C-API functions like NewFromDescr,
+    # so we fall back on copying directly to the underlying buffer
+    # of a new ndarray for other types.
 
     if elem_dtype.kind in b"biufcmMO":
         # type_num is enough to create an array for these dtypes
         ndarray = cnp.PyArray_SimpleNewFromData(1, dims, elem_dtype.type_num, data)
-    else:
-        # dtypes like string & void need a size specified, so can't be used with
-        # SimpleNewFromData. Cython doesn't expose NumPy C-API functions
+    elif not elem_dtype.hasobject:
+        # This covers things like string dtypes and simple compound dtypes,
+        # which can't be used with SimpleNewFromData.
+        # Cython doesn't expose NumPy C-API functions
         # like NewFromDescr, so we'll construct this with a Python function.
         buf = <char[:itemsize * size]> data
         ndarray = np.frombuffer(buf, dtype=elem_dtype)
+    else:
+        # Compound dtypes containing object fields: frombuffer() refuses these,
+        # so we'll fall back to allocating a new array and copying the data in.
+        ndarray = np.empty(size, dtype=elem_dtype)
+        memcpy(PyArray_DATA(ndarray), data, itemsize * size)
+
+        # In this code path, `data`, allocated by hdf5 to hold the v-len data,
+        # will no longer be used since have copied its contents to the ndarray.
+        efree(data)
 
     PyArray_ENABLEFLAGS(ndarray, flags)
     ndarray_obj = <PyObject*>ndarray
@@ -827,22 +851,29 @@ cdef int conv_ndarray2vlen(void* ipt,
         size_t len, nbytes
         PyObject* buf_obj0
         Py_buffer view
+        void* back_buf = NULL
+    try:
+        buf_obj0 = buf_obj[0]
+        ndarray = <cnp.ndarray> buf_obj0
+        len = ndarray.shape[0]
+        nbytes = len * max(H5Tget_size(outtype.id), H5Tget_size(intype.id))
 
-    buf_obj0 = buf_obj[0]
-    ndarray = <cnp.ndarray> buf_obj0
-    len = ndarray.shape[0]
-    nbytes = len * max(H5Tget_size(outtype.id), H5Tget_size(intype.id))
+        data = emalloc(nbytes)
+
+        PyObject_GetBuffer(ndarray, &view, PyBUF_INDIRECT)
+        PyBuffer_ToContiguous(data, &view, view.len, b'C')
+        PyBuffer_Release(&view)
 
-    data = emalloc(nbytes)
+        if needs_bkg_buffer(intype.id, outtype.id):
+            back_buf = emalloc(H5Tget_size(outtype.id)*len)
 
-    PyObject_GetBuffer(ndarray, &view, PyBUF_INDIRECT)
-    PyBuffer_ToContiguous(data, &view, view.len, b'C')
-    PyBuffer_Release(&view)
+        H5Tconvert(intype.id, outtype.id, len, data, back_buf, H5P_DEFAULT)
 
-    H5Tconvert(intype.id, outtype.id, len, data, NULL, H5P_DEFAULT)
+        in_vlen[0].len = len
+        in_vlen[0].ptr = data
 
-    in_vlen[0].len = len
-    in_vlen[0].ptr = data
+    finally:
+        free(back_buf)
 
     return 0