diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9592a71223..faad7cd9e0 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -328,8 +328,7 @@ jobs: -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_ADIOS2=ON \ - -DopenPMD_USE_INVASIVE_TESTS=ON \ - -DMPIEXEC_EXECUTABLE=".github/workflows/mpirun_workaround.sh" + -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 cd build ctest --output-on-failure diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 782acf0bc0..db003484a0 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -40,13 +40,12 @@ jobs: -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_ADIOS2=ON \ - -DopenPMD_USE_INVASIVE_TESTS=ON \ - -DMPIEXEC_EXECUTABLE=".github/workflows/mpirun_workaround.sh" + -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 3 ctest --test-dir build --verbose appleclang14_py: - runs-on: macos-12 + runs-on: macos-13 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/mpirun_workaround.sh b/.github/workflows/mpirun_workaround.sh deleted file mode 100755 index e953b34647..0000000000 --- a/.github/workflows/mpirun_workaround.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -# mpiexec currently seems to have a bug where it tries to parse parameters -# of the launched application when they start with a dash, e.g. -# `mpiexec python ./openpmd-pipe --infile in.bp --outfile out.bp` -# leads to: -# > An unrecognized option was included on the mpiexec command line: -# > -# > Option: --infile -# > -# > Please use the "mpiexec --help" command to obtain a list of all -# > supported options. -# -# This script provides a workaround by putting the called sub-command into -# a script in a temporary file. - -ls="$(which ls)" -mpiexec "$ls" -m \ - && echo "MPIRUN WORKING AGAIN, PLEASE REMOVE WORKAROUND" >&2 \ - && exit 1 \ - || true - -mpirun_args=() - -script_file="$(mktemp)" - -cleanup() { - rm "$script_file" -} -trap cleanup EXIT - -while true; do - case "$1" in - -c | -np | --np | -n | --n ) - mpirun_args+=("$1" "$2") - shift - shift - ;; - *) - break - ;; - esac -done - -echo -e '#!/usr/bin/env bash\n' > "$script_file" -for item in "$@"; do - echo -n "'$item' " >> "$script_file" -done - -chmod +x "$script_file" - -mpirun "${mpirun_args[@]}" "$script_file" diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index efb6d6d762..183bcc4f7e 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -42,7 +42,7 @@ jobs: python3.exe -m pip wheel . if(!$?) { Exit $LASTEXITCODE } - python3.exe -m pip install openPMD_api-0.16.0.dev0-cp39-cp39-win_amd64.whl + python3.exe -m pip install openPMD_api-0.16.1-cp39-cp39-win_amd64.whl if(!$?) { Exit $LASTEXITCODE } python3.exe -c "import openpmd_api as api; print(api.variants)" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 409997c5b8..f37d7f0cb1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,41 @@ Changelog ========= +0.16.1 +------ +**Date:** 2025-01-15 + +New Backends, Extension, Perf. & Memory + +This is the 0.16.0 release but with internal version bumps where they were missing. + +Changes to "0.16.0" +^^^^^^^^^^^^^^^^^^^ + +Bug Fixes +""""""""" + +- HDF5: Delete and re-create attribute when overwriting with diff. type #1697 +- TOUCH IOTask: Avoid setting files as dirty in non-write modes #1704 +- Fix CMake variables for controlling internal dependencies #1678 +- Bump toml11 dependency to ``v4.2.0`` by default #1679 +- Set ``PYBIND11_FINDPYTHON=ON`` #1684 +- Properly check for empty ``HDF5_VERSION`` in CMake #1702 +- Fix ICX build #1690 +- Explicitly specify ADIOS2 components in openPMDConfig.cmake #1693 +- Example 12: Use ``ULONG`` for particle patches #1710 + +Other +""""" + +- Replace deprecated Python unittest API call #1681 +- Remove ``mpirun_workaround.sh`` #1698 +- CI: Upgrade appleclang14 build to MacOS 13 #1703 +- Docs: + + - Fix Shipped Internally #1682 + - Fix leftover ADIOS2 mentions #1683 + 0.16.0 ------ **Date:** 2024-10-07 diff --git a/CITATION.cff b/CITATION.cff index a810842b0c..a0f24d52ea 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -37,7 +37,7 @@ contact: orcid: https://orcid.org/0000-0003-1943-7141 email: axelhuebl@lbl.gov title: "openPMD-api: C++ & Python API for Scientific I/O with openPMD" -version: 0.16.0-dev +version: 0.16.1 repository-code: https://github.com/openPMD/openPMD-api doi: 10.14278/rodare.27 license: LGPL-3.0-or-later diff --git a/CMakeLists.txt b/CMakeLists.txt index a0cc161f4a..9adaeb94ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ # cmake_minimum_required(VERSION 3.22.0) -project(openPMD VERSION 0.16.0) # LANGUAGES CXX +project(openPMD VERSION 0.16.1) # LANGUAGES CXX # the openPMD "markup"/"schema" standard version set(openPMD_STANDARD_VERSION 1.1.0) @@ -314,7 +314,7 @@ endif() string(CONCAT openPMD_HDF5_STATUS "") # version: lower limit if(openPMD_HAVE_HDF5) - if(HDF5_VERSION STREQUAL "") + if("${HDF5_VERSION}" STREQUAL "") message(WARNING "HDF5_VERSION is empty. Now assuming it is 1.8.13 or newer.") else() if(HDF5_VERSION VERSION_LESS 1.8.13) @@ -360,20 +360,24 @@ if(openPMD_HDF5_STATUS) endif() # external library: ADIOS2 (optional) +set(openPMD_REQUIRED_ADIOS2_COMPONENTS CXX) +if(openPMD_HAVE_MPI) + list(APPEND openPMD_REQUIRED_ADIOS2_COMPONENTS MPI) +endif() if(openPMD_USE_ADIOS2 STREQUAL AUTO) - find_package(ADIOS2 2.7.0 CONFIG) + find_package(ADIOS2 2.7.0 CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS}) if(ADIOS2_FOUND) set(openPMD_HAVE_ADIOS2 TRUE) else() set(openPMD_HAVE_ADIOS2 FALSE) endif() elseif(openPMD_USE_ADIOS2) - find_package(ADIOS2 2.7.0 REQUIRED CONFIG) + find_package(ADIOS2 2.7.0 REQUIRED CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS}) set(openPMD_HAVE_ADIOS2 TRUE) else() set(openPMD_HAVE_ADIOS2 FALSE) endif() -# TODO: Check if ADIOS2 is parallel when openPMD_HAVE_MPI is ON +unset(openPMD_REQUIRED_ADIOS2_COMPONENTS) # external library: pybind11 (optional) include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake) diff --git a/README.md b/README.md index 2fc4c7be82..d63cb54530 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Required: * CMake 3.22.0+ * C++17 capable compiler, e.g., g++ 7+, clang 7+, MSVC 19.15+, icpc 19+, icpx -Shipped internally in `share/openPMD/thirdParty/`: +Shipped internally (downloaded by CMake unless `openPMD_SUPERBUILD=OFF` is set): * [Catch2](https://github.com/catchorg/Catch2) 2.13.10+ ([BSL-1.0](https://github.com/catchorg/Catch2/blob/master/LICENSE.txt)) * [pybind11](https://github.com/pybind/pybind11) 2.13.0+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE)) * [NLohmann-JSON](https://github.com/nlohmann/json) 3.9.1+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT)) @@ -304,7 +304,7 @@ export CMAKE_PREFIX_PATH=$HOME/somepath:$CMAKE_PREFIX_PATH Use the following lines in your project's `CMakeLists.txt`: ```cmake # supports: COMPONENTS MPI NOMPI HDF5 ADIOS2 -find_package(openPMD 0.16.0 CONFIG) +find_package(openPMD 0.16.1 CONFIG) if(openPMD_FOUND) target_link_libraries(YourTarget PRIVATE openPMD::openPMD) @@ -332,7 +332,7 @@ set(openPMD_INSTALL OFF) # or instead use: set(openPMD_USE_PYTHON OFF) FetchContent_Declare(openPMD GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" - GIT_TAG "0.16.0") + GIT_TAG "0.16.1") FetchContent_MakeAvailable(openPMD) ``` diff --git a/cmake/dependencies/catch.cmake b/cmake/dependencies/catch.cmake index 956d66493e..bc7a3dcc89 100644 --- a/cmake/dependencies/catch.cmake +++ b/cmake/dependencies/catch.cmake @@ -1,49 +1,49 @@ function(find_catch2) if(TARGET Catch2::Catch2) message(STATUS "Catch2::Catch2 target already imported") - elseif(openPMD_catch_src) - message(STATUS "Compiling local Catch2 ...") - message(STATUS "Catch2 source path: ${openPMD_catch_src}") - if(NOT IS_DIRECTORY ${openPMD_catch_src}) - message(FATAL_ERROR "Specified directory openPMD_catch_src='${openPMD_catch_src}' does not exist!") - endif() - elseif(openPMD_catch_tar) - message(STATUS "Downloading Catch2 ...") - message(STATUS "Catch2 source: ${openPMD_catch_tar}") elseif(openPMD_USE_INTERNAL_CATCH) - message(STATUS "Downloading Catch2 ...") - message(STATUS "Catch2 repository: ${openPMD_catch_repo} (${openPMD_catch_branch})") + if(openPMD_catch_src) + message(STATUS "Compiling local Catch2 ...") + message(STATUS "Catch2 source path: ${openPMD_catch_src}") + if(NOT IS_DIRECTORY ${openPMD_catch_src}) + message(FATAL_ERROR "Specified directory openPMD_catch_src='${openPMD_catch_src}' does not exist!") + endif() + elseif(openPMD_catch_tar) + message(STATUS "Downloading Catch2 ...") + message(STATUS "Catch2 source: ${openPMD_catch_tar}") + elseif(openPMD_catch_branch) + message(STATUS "Downloading Catch2 ...") + message(STATUS "Catch2 repository: ${openPMD_catch_repo} (${openPMD_catch_branch})") + endif() endif() if(TARGET Catch2::Catch2) # nothing to do, target already exists in the superbuild - elseif(openPMD_USE_INTERNAL_CATCH OR openPMD_catch_src OR openPMD_catch_tar) - if(openPMD_catch_src) - add_subdirectory(${openPMD_catch_src} _deps/localCatch2-build/) + elseif(openPMD_USE_INTERNAL_CATCH AND openPMD_catch_src) + add_subdirectory(${openPMD_catch_src} _deps/localCatch2-build/) + elseif(openPMD_USE_INTERNAL_CATCH AND (openPMD_catch_tar OR openPMD_catch_branch)) + include(FetchContent) + if(openPMD_catch_tar) + FetchContent_Declare(fetchedCatch2 + URL ${openPMD_catch_tar} + URL_HASH ${openPMD_catch_tar_hash} + BUILD_IN_SOURCE OFF + ) else() - include(FetchContent) - if(openPMD_catch_tar) - FetchContent_Declare(fetchedCatch2 - URL ${openPMD_catch_tar} - URL_HASH ${openPMD_catch_tar_hash} - BUILD_IN_SOURCE OFF - ) - else() - FetchContent_Declare(fetchedCatch2 - GIT_REPOSITORY ${openPMD_catch_repo} - GIT_TAG ${openPMD_catch_branch} - BUILD_IN_SOURCE OFF - ) - endif() - FetchContent_MakeAvailable(fetchedCatch2) - - # advanced fetch options - mark_as_advanced(FETCHCONTENT_BASE_DIR) - mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) - mark_as_advanced(FETCHCONTENT_QUIET) - #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDCatch2) - mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) - #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDCatch2) + FetchContent_Declare(fetchedCatch2 + GIT_REPOSITORY ${openPMD_catch_repo} + GIT_TAG ${openPMD_catch_branch} + BUILD_IN_SOURCE OFF + ) endif() + FetchContent_MakeAvailable(fetchedCatch2) + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDCatch2) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDCatch2) elseif(NOT openPMD_USE_INTERNAL_CATCH) find_package(Catch2 2.13.10 CONFIG REQUIRED) message(STATUS "Catch2: Found version '${Catch2_VERSION}'") diff --git a/cmake/dependencies/json.cmake b/cmake/dependencies/json.cmake index 8b22eb80f8..751ffc87f5 100644 --- a/cmake/dependencies/json.cmake +++ b/cmake/dependencies/json.cmake @@ -1,49 +1,49 @@ function(find_json) if(TARGET nlohmann_json::nlohmann_json) message(STATUS "nlohmann_json::nlohmann_json target already imported") - elseif(openPMD_json_src) - message(STATUS "Compiling local nlohmann_json ...") - message(STATUS "nlohmann_json source path: ${openPMD_json_src}") - if(NOT IS_DIRECTORY ${openPMD_json_src}) - message(FATAL_ERROR "Specified directory openPMD_json_src='${openPMD_json_src}' does not exist!") - endif() - elseif(openPMD_json_tar) - message(STATUS "Downloading nlohmann_json ...") - message(STATUS "nlohmann_json source: ${openPMD_json_tar}") elseif(openPMD_USE_INTERNAL_JSON) - message(STATUS "Downloading nlohmann_json ...") - message(STATUS "nlohmann_json repository: ${openPMD_json_repo} (${openPMD_json_branch})") + if(openPMD_json_src) + message(STATUS "Compiling local nlohmann_json ...") + message(STATUS "nlohmann_json source path: ${openPMD_json_src}") + if(NOT IS_DIRECTORY ${openPMD_json_src}) + message(FATAL_ERROR "Specified directory openPMD_json_src='${openPMD_json_src}' does not exist!") + endif() + elseif(openPMD_json_tar) + message(STATUS "Downloading nlohmann_json ...") + message(STATUS "nlohmann_json source: ${openPMD_json_tar}") + elseif(openPMD_json_branch) + message(STATUS "Downloading nlohmann_json ...") + message(STATUS "nlohmann_json repository: ${openPMD_json_repo} (${openPMD_json_branch})") + endif() endif() if(TARGET nlohmann_json::nlohmann_json) # nothing to do, target already exists in the superbuild - elseif(openPMD_USE_INTERNAL_JSON OR openPMD_json_src OR openPMD_json_tar) - if(openPMD_json_src) - add_subdirectory(${openPMD_json_src} _deps/localnlohmann_json-build/) + elseif(openPMD_USE_INTERNAL_JSON AND openPMD_json_src) + add_subdirectory(${openPMD_json_src} _deps/localnlohmann_json-build/) + elseif(openPMD_USE_INTERNAL_JSON AND (openPMD_json_tar OR openPMD_json_branch)) + include(FetchContent) + if(openPMD_json_tar) + FetchContent_Declare(fetchednlohmann_json + URL ${openPMD_json_tar} + URL_HASH ${openPMD_json_tar_hash} + BUILD_IN_SOURCE OFF + ) else() - include(FetchContent) - if(openPMD_json_tar) - FetchContent_Declare(fetchednlohmann_json - URL ${openPMD_json_tar} - URL_HASH ${openPMD_json_tar_hash} - BUILD_IN_SOURCE OFF - ) - else() - FetchContent_Declare(fetchednlohmann_json - GIT_REPOSITORY ${openPMD_json_repo} - GIT_TAG ${openPMD_json_branch} - BUILD_IN_SOURCE OFF - ) - endif() - FetchContent_MakeAvailable(fetchednlohmann_json) - - # advanced fetch options - mark_as_advanced(FETCHCONTENT_BASE_DIR) - mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) - mark_as_advanced(FETCHCONTENT_QUIET) - #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDnlohmann_json) - mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) - #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDnlohmann_json) + FetchContent_Declare(fetchednlohmann_json + GIT_REPOSITORY ${openPMD_json_repo} + GIT_TAG ${openPMD_json_branch} + BUILD_IN_SOURCE OFF + ) endif() + FetchContent_MakeAvailable(fetchednlohmann_json) + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDnlohmann_json) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDnlohmann_json) elseif(NOT openPMD_USE_INTERNAL_JSON) find_package(nlohmann_json 3.9.1 CONFIG REQUIRED) message(STATUS "nlohmann_json: Found version '${nlohmann_json_VERSION}'") diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index 742aafa7dc..70899de816 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -1,49 +1,54 @@ function(find_pybind11) if(TARGET pybind11::module) message(STATUS "pybind11::module target already imported") - elseif(openPMD_pybind11_src) - message(STATUS "Compiling local pybind11 ...") - message(STATUS "pybind11 source path: ${openPMD_pybind11_src}") - if(NOT IS_DIRECTORY ${openPMD_pybind11_src}) - message(FATAL_ERROR "Specified directory openPMD_pybind11_src='${openPMD_pybind11_src}' does not exist!") - endif() - elseif(openPMD_pybind11_tar) - message(STATUS "Downloading pybind11 ...") - message(STATUS "pybind11 source: ${openPMD_pybind11_tar}") elseif(openPMD_USE_INTERNAL_PYBIND11) - message(STATUS "Downloading pybind11 ...") - message(STATUS "pybind11 repository: ${openPMD_pybind11_repo} (${openPMD_pybind11_branch})") + if(openPMD_pybind11_src) + message(STATUS "Compiling local pybind11 ...") + message(STATUS "pybind11 source path: ${openPMD_pybind11_src}") + if(NOT IS_DIRECTORY ${openPMD_pybind11_src}) + message(FATAL_ERROR "Specified directory openPMD_pybind11_src='${openPMD_pybind11_src}' does not exist!") + endif() + elseif(openPMD_pybind11_tar) + message(STATUS "Downloading pybind11 ...") + message(STATUS "pybind11 source: ${openPMD_pybind11_tar}") + elseif(openPMD_pybind11_branch) + message(STATUS "Downloading pybind11 ...") + message(STATUS "pybind11 repository: ${openPMD_pybind11_repo} (${openPMD_pybind11_branch})") + endif() endif() + + # rely on our find_package(Python ...) call + # https://pybind11.readthedocs.io/en/stable/compiling.html#modules-with-cmake + set(PYBIND11_FINDPYTHON ON) + if(TARGET pybind11::module) # nothing to do, target already exists in the superbuild - elseif(openPMD_USE_INTERNAL_PYBIND11 OR openPMD_pybind11_src) - if(openPMD_pybind11_src) - add_subdirectory(${openPMD_pybind11_src} _deps/localpybind11-build/) - else() - include(FetchContent) - if(openPMD_pybind11_tar) - FetchContent_Declare(fetchedpybind11 - URL ${openPMD_pybind11_tar} - URL_HASH ${openPMD_pybind11_tar_hash} - BUILD_IN_SOURCE OFF - ) - else() - FetchContent_Declare(fetchedpybind11 - GIT_REPOSITORY ${openPMD_pybind11_repo} - GIT_TAG ${openPMD_pybind11_branch} + elseif(openPMD_USE_INTERNAL_PYBIND11 AND openPMD_pybind11_src) + add_subdirectory(${openPMD_pybind11_src} _deps/localpybind11-build/) + elseif(openPMD_USE_INTERNAL_PYBIND11 AND (openPMD_pybind11_tar OR openPMD_pybind11_branch)) + include(FetchContent) + if(openPMD_pybind11_tar) + FetchContent_Declare(fetchedpybind11 + URL ${openPMD_pybind11_tar} + URL_HASH ${openPMD_pybind11_tar_hash} BUILD_IN_SOURCE OFF - ) - endif() - FetchContent_MakeAvailable(fetchedpybind11) - - # advanced fetch options - mark_as_advanced(FETCHCONTENT_BASE_DIR) - mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) - mark_as_advanced(FETCHCONTENT_QUIET) - mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDpybind11) - mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) - mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) + ) + else() + FetchContent_Declare(fetchedpybind11 + GIT_REPOSITORY ${openPMD_pybind11_repo} + GIT_TAG ${openPMD_pybind11_branch} + BUILD_IN_SOURCE OFF + ) endif() + FetchContent_MakeAvailable(fetchedpybind11) + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDpybind11) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) elseif(NOT openPMD_USE_INTERNAL_PYBIND11) if(openPMD_USE_PYTHON STREQUAL AUTO) find_package(pybind11 2.13.0 CONFIG) diff --git a/cmake/dependencies/toml11.cmake b/cmake/dependencies/toml11.cmake index 6ad9990db2..c1f8832a81 100644 --- a/cmake/dependencies/toml11.cmake +++ b/cmake/dependencies/toml11.cmake @@ -1,49 +1,49 @@ function(find_toml11) if(TARGET toml11::toml11) message(STATUS "toml11::toml11 target already imported") - elseif(openPMD_toml11_src) - message(STATUS "Compiling local toml11 ...") - message(STATUS "toml11 source path: ${openPMD_toml11_src}") - if(NOT IS_DIRECTORY ${openPMD_toml11_src}) - message(FATAL_ERROR "Specified directory openPMD_toml11_src='${openPMD_toml11_src}' does not exist!") - endif() - elseif(openPMD_toml11_tar) - message(STATUS "Downloading toml11 ...") - message(STATUS "toml11 source: ${openPMD_toml11_tar}") elseif(openPMD_USE_INTERNAL_TOML11) - message(STATUS "Downloading toml11 ...") - message(STATUS "toml11 repository: ${openPMD_toml11_repo} (${openPMD_toml11_branch})") + if(openPMD_toml11_src) + message(STATUS "Compiling local toml11 ...") + message(STATUS "toml11 source path: ${openPMD_toml11_src}") + if(NOT IS_DIRECTORY ${openPMD_toml11_src}) + message(FATAL_ERROR "Specified directory openPMD_toml11_src='${openPMD_toml11_src}' does not exist!") + endif() + elseif(openPMD_toml11_tar) + message(STATUS "Downloading toml11 ...") + message(STATUS "toml11 source: ${openPMD_toml11_tar}") + elseif(openPMD_toml11_branch) + message(STATUS "Downloading toml11 ...") + message(STATUS "toml11 repository: ${openPMD_toml11_repo} (${openPMD_toml11_branch})") + endif() endif() if(TARGET toml11::toml11) # nothing to do, target already exists in the superbuild - elseif(openPMD_USE_INTERNAL_TOML11 OR openPMD_toml11_src) - if(openPMD_toml11_src) - add_subdirectory(${openPMD_toml11_src} _deps/localtoml11-build/) - else() - include(FetchContent) - if(openPMD_toml11_tar) - FetchContent_Declare(fetchedtoml11 - URL ${openPMD_toml11_tar} - URL_HASH ${openPMD_toml11_tar_hash} - BUILD_IN_SOURCE OFF - ) - else() - FetchContent_Declare(fetchedtoml11 - GIT_REPOSITORY ${openPMD_toml11_repo} - GIT_TAG ${openPMD_toml11_branch} + elseif(openPMD_USE_INTERNAL_TOML11 AND openPMD_toml11_src) + add_subdirectory(${openPMD_toml11_src} _deps/localtoml11-build/) + elseif(openPMD_USE_INTERNAL_TOML11 AND (openPMD_toml11_tar OR openPMD_toml11_branch)) + include(FetchContent) + if(openPMD_toml11_tar) + FetchContent_Declare(fetchedtoml11 + URL ${openPMD_toml11_tar} + URL_HASH ${openPMD_toml11_tar_hash} BUILD_IN_SOURCE OFF - ) - endif() - FetchContent_MakeAvailable(fetchedtoml11) - - # advanced fetch options - mark_as_advanced(FETCHCONTENT_BASE_DIR) - mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) - mark_as_advanced(FETCHCONTENT_QUIET) - #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDtoml11) - mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) - #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDtoml11) + ) + else() + FetchContent_Declare(fetchedtoml11 + GIT_REPOSITORY ${openPMD_toml11_repo} + GIT_TAG ${openPMD_toml11_branch} + BUILD_IN_SOURCE OFF + ) endif() + FetchContent_MakeAvailable(fetchedtoml11) + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDtoml11) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDtoml11) elseif(NOT openPMD_USE_INTERNAL_TOML11) # toml11 4.0 was a breaking change. This is reflected in the library's CMake # logic: version 4.0 is not accepted by a call to find_package(toml11 3.7). @@ -64,10 +64,10 @@ set(openPMD_toml11_src "" "Local path to toml11 source directory (preferred if set)") # tarball fetcher -set(openPMD_toml11_tar "https://github.com/ToruNiina/toml11/archive/refs/tags/v3.7.1.tar.gz" +set(openPMD_toml11_tar "https://github.com/ToruNiina/toml11/archive/refs/tags/v4.2.0.tar.gz" CACHE STRING "Remote tarball link to pull and build toml11 from if(openPMD_USE_INTERNAL_TOML11)") -set(openPMD_toml11_tar_hash "SHA256=afeaa9aa0416d4b6b2cd3897ca55d9317084103077b32a852247d8efd4cf6068" +set(openPMD_toml11_tar_hash "SHA256=9287971cd4a1a3992ef37e7b95a3972d1ae56410e7f8e3f300727ab1d6c79c2c" CACHE STRING "Hash checksum of the tarball of toml11 if(openPMD_USE_INTERNAL_TOML11)") diff --git a/docs/Doxyfile b/docs/Doxyfile index 471c3cef28..16f734aa4f 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -1,7 +1,6 @@ PROJECT_NAME = "openPMD-api" XML_OUTPUT = xml INPUT = ../src ../include ../README.md -#EXCLUDE_PATTERNS = *CommonADIOS1IOHandler.cpp # TAGFILES += "cppreference-doxygen-web.tag.xml=http://en.cppreference.com/w/" BUILTIN_STL_SUPPORT = YES diff --git a/docs/source/conf.py b/docs/source/conf.py index 668ef4c8ae..f93e0a2763 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -86,9 +86,9 @@ # built documents. # # The short X.Y version. -version = u'0.16.0' +version = u'0.16.1' # The full version, including alpha/beta/rc tags. -release = u'0.16.0-dev' +release = u'0.16.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index c0318aa683..87667f0efb 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -15,7 +15,7 @@ Required Shipped internally ------------------ -The following libraries are shipped internally in ``share/openPMD/thirdParty/`` for convenience: +The following libraries are downloaded by CMake for convenience, unless ``openPMD_SUPERBUILD=OFF`` is set: * `Catch2 `_ 2.13.10+ (`BSL-1.0 `__) * `pybind11 `_ 2.13.0+ (`new BSD `_) diff --git a/docs/source/dev/linking.rst b/docs/source/dev/linking.rst index 191612f541..5048ddbf1d 100644 --- a/docs/source/dev/linking.rst +++ b/docs/source/dev/linking.rst @@ -23,7 +23,7 @@ Use the following lines in your project's ``CMakeLists.txt``: .. code-block:: cmake # supports: COMPONENTS MPI NOMPI HDF5 ADIOS2 - find_package(openPMD 0.16.0 CONFIG) + find_package(openPMD 0.16.1 CONFIG) if(openPMD_FOUND) target_link_libraries(YourTarget PRIVATE openPMD::openPMD) @@ -53,7 +53,7 @@ Just replace the ``add_subdirectory`` call with: set(openPMD_USE_PYTHON OFF) FetchContent_Declare(openPMD GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" - GIT_TAG "0.16.0") + GIT_TAG "0.16.1") FetchContent_MakeAvailable(openPMD) diff --git a/docs/source/index.rst b/docs/source/index.rst index 5fc4475364..e38b87d486 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -44,7 +44,7 @@ openPMD-api version supported openPMD standard versions ======================== =================================== ``2.0.0+`` ``2.0.0+`` (not released yet) ``1.0.0+`` ``1.0.1-1.1.0`` (not released yet) -``0.13.1-0.15.1`` (beta) ``1.0.0-1.1.0`` +``0.13.1-0.16.1`` (beta) ``1.0.0-1.1.0`` ``0.1.0-0.12.0`` (alpha) ``1.0.0-1.1.0`` ======================== =================================== diff --git a/docs/source/maintenance/release_channels.rst b/docs/source/maintenance/release_channels.rst index bd40f7f784..9c43711013 100644 --- a/docs/source/maintenance/release_channels.rst +++ b/docs/source/maintenance/release_channels.rst @@ -39,7 +39,7 @@ Brew We maintain a `homebrew tap `_ for `openPMD `_. Provides the C++ and Python API for users. Supports OSX and Linux. -Its source-only Formula for the latest release includes (Open)MPI support and lacks the ADIOS1 backend. +Its source-only Formula for the latest release includes (Open)MPI support. Example workflow for a new release: diff --git a/examples/12_span_write.cpp b/examples/12_span_write.cpp index e2efcffdaa..201e062ff9 100644 --- a/examples/12_span_write.cpp +++ b/examples/12_span_write.cpp @@ -44,9 +44,11 @@ void span_write(std::string const &filename) } for (auto record : {"numParticlesOffset", "numParticles"}) { - patches[record].resetDataset({Datatype::INT, {1}}); - *patches[record].storeChunk({0}, {1}).currentBuffer().data() = - 42; + patches[record].resetDataset({Datatype::ULONG, {1}}); + *patches[record] + .storeChunk({0}, {1}) + .currentBuffer() + .data() = 42; } Record electronPositions = iteration.particles["e"]["position"]; diff --git a/examples/13_write_dynamic_configuration.cpp b/examples/13_write_dynamic_configuration.cpp index 73ce0be095..b480cb2f00 100644 --- a/examples/13_write_dynamic_configuration.cpp +++ b/examples/13_write_dynamic_configuration.cpp @@ -39,9 +39,6 @@ iteration_encoding = "group_based" # The following is only relevant in read mode defer_iteration_parsing = true -[adios1.dataset] -transform = "blosc:compressor=zlib,shuffle=bit,lvl=5;nometa" - [adios2.engine] type = "bp4" diff --git a/examples/13_write_dynamic_configuration.py b/examples/13_write_dynamic_configuration.py index f3e1e63bd5..a0f7b0c213 100644 --- a/examples/13_write_dynamic_configuration.py +++ b/examples/13_write_dynamic_configuration.py @@ -25,9 +25,6 @@ # The following is only relevant in read mode defer_iteration_parsing = true -[adios1.dataset] -transform = "blosc:compressor=zlib,shuffle=bit,lvl=5;nometa" - [adios2.engine] type = "bp4" diff --git a/examples/7_extended_write_serial.cpp b/examples/7_extended_write_serial.cpp index 6b69635fc6..63d9cb38ff 100644 --- a/examples/7_extended_write_serial.cpp +++ b/examples/7_extended_write_serial.cpp @@ -107,11 +107,6 @@ int main() auto d = io::Dataset(dtype, io::Extent{2, 5}); std::string datasetConfig = R"END( { - "adios1": { - "dataset": { - "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" - } - }, "adios2": { "dataset": { "operators": [ diff --git a/examples/7_extended_write_serial.py b/examples/7_extended_write_serial.py index d9b50693cb..f63a5c8371 100755 --- a/examples/7_extended_write_serial.py +++ b/examples/7_extended_write_serial.py @@ -106,11 +106,6 @@ # written to disk d = Dataset(partial_mesh.dtype, extent=[2, 5]) dataset_config = { - "adios1": { - "dataset": { - "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" - } - }, "adios2": { "dataset": { "operators": [{ diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 58b07bd48a..cee50f9baf 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -259,7 +259,7 @@ class Container : virtual public Attributable void swap(Container &other) { - container().swap(other.m_container); + container().swap(other.container()); } mapped_type &at(key_type const &key) diff --git a/include/openPMD/version.hpp b/include/openPMD/version.hpp index 9f32154d92..ead912a23c 100644 --- a/include/openPMD/version.hpp +++ b/include/openPMD/version.hpp @@ -29,7 +29,7 @@ */ #define OPENPMDAPI_VERSION_MAJOR 0 #define OPENPMDAPI_VERSION_MINOR 16 -#define OPENPMDAPI_VERSION_PATCH 0 +#define OPENPMDAPI_VERSION_PATCH 1 #define OPENPMDAPI_VERSION_LABEL "" /** @} */ diff --git a/new_version.py b/new_version.py index 59f2e4e49d..a261c99add 100755 --- a/new_version.py +++ b/new_version.py @@ -14,13 +14,11 @@ # Maintainer Inputs ########################################################### -print( - """Hi there, this is an openPMD maintainer tool to update the source +print("""Hi there, this is an openPMD maintainer tool to update the source code of openPMD-api to a new version. For it to work, you need write access on the source directory and you should be working in a clean git branch without ongoing -rebase/merge/conflict resolves and without unstaged changes.""" -) +rebase/merge/conflict resolves and without unstaged changes.""") # check source dir # REPO_DIR = Path(__file__).parent.parent.parent.absolute() @@ -40,6 +38,7 @@ VERSION_STR = f"{MAJOR}.{MINOR}.{PATCH}" VERSION_STR_SUFFIX = VERSION_STR + (f"-{SUFFIX}" if SUFFIX else "") +VERSION_STR_SUFFIX_WITH_DOT = VERSION_STR + (f".{SUFFIX}" if SUFFIX else "") print() print(f"Your new version is: {VERSION_STR_SUFFIX}") @@ -51,20 +50,41 @@ for line in f: match = re.search(r"find_package.*openPMD *([^ ]*) *CONFIG\).*", line) if match: - OLD_VERSION_STR = match.group(1) + OLD_VERSION_STR_README = match.group(1) + break + +with open(str(REPO_DIR.joinpath("CMakeLists.txt")), encoding="utf-8") as f: + for line in f: + match = re.search(r"project\(openPMD *VERSION *(.*)\)", line) + if match: + OLD_VERSION_STR_CMAKE = match.group(1) break +OLD_VERSION_TAG = "" +with open(str(REPO_DIR.joinpath("include/openPMD/version.hpp")), + encoding="utf-8") as f: + for line in f: + match = re.search(r'#define OPENPMDAPI_VERSION_LABEL "([^"]+)"', line) + if match: + OLD_VERSION_TAG = match.group(1) + break + +OLD_VERSION_SUFFIX = f"(-{OLD_VERSION_TAG})?" if OLD_VERSION_TAG else "" +# The order of the alternatives is important, since the Regex parser +# should greedily include the old version suffix +OLD_VERSION_STR = \ + f"({re.escape(OLD_VERSION_STR_CMAKE)}{OLD_VERSION_SUFFIX})" + \ + f"|({re.escape(OLD_VERSION_STR_README)})" + print(f"The old version is: {OLD_VERSION_STR}") print() - REPLY = input("Is this information correct? Will now start updating! [y/N] ") print() if REPLY not in ["Y", "y", ""]: print("You did not confirm with 'y', aborting.") sys.exit(1) - # Ask for new ################################################################# print("""We will now run a few sed commands on your source directory.\n""") @@ -86,18 +106,56 @@ f.write(cmakelists_content) -def generic_replace(filename): +def generic_replace(filename, previous, after): filename = str(REPO_DIR.joinpath(filename)) with open(filename, encoding="utf-8") as f: content = f.read() - content = re.sub(re.escape(OLD_VERSION_STR), VERSION_STR, content) + content = re.sub(previous, after, content, flags=re.MULTILINE) with open(filename, "w", encoding="utf-8") as f: f.write(content) -for file in ["docs/source/dev/linking.rst", "README.md"]: - generic_replace(file) +for file in [ + "docs/source/dev/linking.rst", + "README.md", +]: + generic_replace(file, previous=OLD_VERSION_STR, after=VERSION_STR) + +for file in ["CITATION.cff", "test/SerialIOTest.cpp"]: + generic_replace(file, previous=OLD_VERSION_STR, after=VERSION_STR_SUFFIX) + +generic_replace( + "docs/source/index.rst", + previous=r"``0.13.1-[^`]*``", + after=f"``0.13.1-{VERSION_STR}``", +) +setup_py_path = str(REPO_DIR.joinpath("setup.py")) +with open(setup_py_path, encoding="utf-8") as f: + for line in f: + match = re.search(r"version='([^']+)',", line) + if match: + PREVIOUS_PIP_VERSION = match.group(1) + break +generic_replace("setup.py", + previous=PREVIOUS_PIP_VERSION, + after=VERSION_STR_SUFFIX_WITH_DOT) +generic_replace( + ".github/workflows/windows.yml", + previous=f"{PREVIOUS_PIP_VERSION}0?", + after=(f"{VERSION_STR_SUFFIX_WITH_DOT}0" + if SUFFIX else VERSION_STR_SUFFIX_WITH_DOT), +) +generic_replace( + "docs/source/conf.py", + previous=r"^version.*=.*", + after=f"version = u'{VERSION_STR}'", +) +generic_replace( + "docs/source/conf.py", + previous=r"^release.*=.*", + after=f"release = u'{VERSION_STR_SUFFIX}'", +) version_hpp_path = str(REPO_DIR.joinpath("include/openPMD/version.hpp")) with open(version_hpp_path, encoding="utf-8") as f: @@ -122,8 +180,6 @@ def replace(key, value): # Epilogue #################################################################### -print( - """Done. Please check your source, e.g. via +print("""Done. Please check your source, e.g. via git diff -now and commit the changes if no errors occurred.""" -) +now and commit the changes if no errors occurred.""") diff --git a/openPMDConfig.cmake.in b/openPMDConfig.cmake.in index 3f6902e503..4cf4ff1732 100644 --- a/openPMDConfig.cmake.in +++ b/openPMDConfig.cmake.in @@ -31,7 +31,12 @@ set(openPMD_HDF5_FOUND ${openPMD_HAVE_HDF5}) set(openPMD_HAVE_ADIOS2 @openPMD_HAVE_ADIOS2@) if(openPMD_HAVE_ADIOS2) - find_dependency(ADIOS2) + set(openPMD_REQUIRED_ADIOS2_COMPONENTS CXX) + if(openPMD_HAVE_MPI) + LIST(APPEND openPMD_REQUIRED_ADIOS2_COMPONENTS MPI) + endif() + find_dependency(ADIOS2 COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS}) + unset(openPMD_REQUIRED_ADIOS2_COMPONENTS) endif() set(openPMD_ADIOS2_FOUND ${openPMD_HAVE_ADIOS2}) diff --git a/setup.py b/setup.py index 1c4e7476d3..b368be19a4 100644 --- a/setup.py +++ b/setup.py @@ -44,12 +44,19 @@ def build_extension(self, ext): if not extdir.endswith(os.path.sep): extdir += os.path.sep + pyv = sys.version_info cmake_args = [ + # Python: use the calling interpreter in CMake + # https://cmake.org/cmake/help/latest/module/FindPython.html#hints + # https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-version-selection + '-DPython_ROOT_DIR=' + sys.prefix, + f'-DPython_FIND_VERSION={pyv.major}.{pyv.minor}.{pyv.micro}', + '-DPython_FIND_VERSION_EXACT=TRUE', + '-DPython_FIND_STRATEGY=LOCATION', '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + os.path.join(extdir, "openpmd_api"), # '-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=' + extdir, '-DopenPMD_PYTHON_OUTPUT_DIRECTORY=' + extdir, - '-DPython_EXECUTABLE=' + sys.executable, '-DopenPMD_USE_PYTHON:BOOL=ON', # variants '-DopenPMD_USE_MPI:BOOL=' + openPMD_USE_MPI, @@ -174,7 +181,7 @@ def build_extension(self, ext): setup( name='openPMD-api', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version='0.16.0.dev', + version='0.16.1', author='Axel Huebl, Franz Poeschel, Fabian Koller, Junmin Gu', author_email='axelhuebl@lbl.gov, f.poeschel@hzdr.de', maintainer='Axel Huebl', diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2946b047f0..05cf7cd689 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -772,7 +772,7 @@ void ADIOS2IOHandlerImpl::createDataset( if (parameters.joinedDimension.has_value()) { error::throwOperationUnsupportedInBackend( - "ADIOS1", "Joined Arrays require ADIOS2 >= v2.9"); + "ADIOS2", "Joined Arrays require ADIOS2 >= v2.9"); } #endif if (!writable->written) @@ -1520,7 +1520,15 @@ void ADIOS2IOHandlerImpl::touch( Writable *writable, Parameter const &) { auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); - m_dirty.emplace(std::move(file)); + if (access::write(m_handler->m_backendAccess)) + { + m_dirty.emplace(std::move(file)); + } + else if (m_fileData.find(file) == m_fileData.end()) + { + throw error::Internal( + "ADIOS2: Tried activating a file that is not open."); + } } adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 32a9e80d74..874768adbe 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -471,7 +471,7 @@ void HDF5IOHandlerImpl::createDataset( if (parameters.joinedDimension.has_value()) { error::throwOperationUnsupportedInBackend( - "ADIOS1", "Joined Arrays currently only supported in ADIOS2"); + "HDF5", "Joined Arrays currently only supported in ADIOS2"); } if (!writable->written) @@ -1694,8 +1694,7 @@ void HDF5IOHandlerImpl::writeAttribute( "[HDF5] Internal error: Failed to get HDF5 datatype during attribute " "write"); std::string name = parameters.name; - if (H5Aexists(node_id, name.c_str()) == 0) - { + auto create_attribute_anew = [&]() { hid_t dataspace = getH5DataSpace(att); VERIFY( dataspace >= 0, @@ -1717,14 +1716,47 @@ void HDF5IOHandlerImpl::writeAttribute( status == 0, "[HDF5] Internal error: Failed to close HDF5 dataspace during " "attribute write"); - } - else + }; + if (H5Aexists(node_id, name.c_str()) != 0) { attribute_id = H5Aopen(node_id, name.c_str(), H5P_DEFAULT); VERIFY( - node_id >= 0, + attribute_id >= 0, "[HDF5] Internal error: Failed to open HDF5 attribute during " "attribute write"); + /* + * Only reuse the old attribute if it had the same type. + */ + hid_t type_id = H5Aget_type(attribute_id); + VERIFY( + type_id >= 0, + "[HDF5] Internal error: Failed to inquire HDF5 attribute type " + "during attribute write"); + auto equal = H5Tequal(type_id, dataType); + VERIFY( + equal >= 0, + "[HDF5] Internal error: Failed to compare HDF5 attribute types " + "during attribute write"); + if (equal == 0) // unequal + { + status = H5Aclose(attribute_id); + VERIFY( + status == 0, + "[HDF5] Internal error: Failed to close previous HDF5 " + "attribute " + "during attribute write"); + status = H5Adelete(node_id, name.c_str()); + VERIFY( + status == 0, + "[HDF5] Internal error: Failed to delete previous HDF5 " + "attribute " + "during attribute write"); + create_attribute_anew(); + } + } + else + { + create_attribute_anew(); } using DT = Datatype; diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index e06aa36ed8..b1046c4602 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -25,6 +25,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/Access.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Memory.hpp" @@ -158,6 +159,11 @@ JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; std::future JSONIOHandlerImpl::flush() { AbstractIOHandlerImpl::flush(); + if (access::readOnly(m_handler->m_backendAccess) && !m_dirty.empty()) + { + throw error::Internal( + "JSON backend: Cannot have dirty files in read-only modes."); + } for (auto const &file : m_dirty) { putJsonContents(file, false); @@ -283,7 +289,7 @@ void JSONIOHandlerImpl::createDataset( if (parameter.joinedDimension.has_value()) { error::throwOperationUnsupportedInBackend( - "ADIOS1", "Joined Arrays currently only supported in ADIOS2"); + "JSON", "Joined Arrays currently only supported in ADIOS2"); } if (!writable->written) @@ -1044,7 +1050,15 @@ void JSONIOHandlerImpl::touch( Writable *writable, Parameter const &) { auto file = refreshFileFromParent(writable); - m_dirty.emplace(std::move(file)); + if (access::write(m_handler->m_backendAccess)) + { + m_dirty.emplace(std::move(file)); + } + else if (m_jsonVals.find(file) == m_jsonVals.end()) + { + throw error::Internal( + "ADIOS2: Tried activating a file that is not open."); + } } auto JSONIOHandlerImpl::getFilehandle(File const &fileName, Access access) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 7323a32582..834b35ca02 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -6363,7 +6363,7 @@ void adios2_bp5_no_steps(bool usesteps) IO.DefineAttribute("/openPMD", std::string("1.1.0")); IO.DefineAttribute("/openPMDextension", uint32_t(0)); IO.DefineAttribute("/software", std::string("openPMD-api")); - IO.DefineAttribute("/softwareVersion", std::string("0.16.0-dev")); + IO.DefineAttribute("/softwareVersion", std::string("0.16.1")); IO.DefineAttribute("/data/0/dt", double(1)); IO.DefineAttribute( @@ -7477,6 +7477,8 @@ void groupbased_read_write(std::string const &ext) E_x.setAttribute("updated_in_run", 0); E_y.setAttribute("updated_in_run", 0); + E_y.setAttribute("changed_datatype_in_run", 0); + write.close(); } { @@ -7495,6 +7497,8 @@ void groupbased_read_write(std::string const &ext) E_x.setAttribute("updated_in_run", 1); E_y.setAttribute("updated_in_run", 1); + E_y.setAttribute("changed_datatype_in_run", "one"); + write.close(); } { @@ -7506,6 +7510,9 @@ void groupbased_read_write(std::string const &ext) REQUIRE(E_x_0_fromRun0.getAttribute("updated_in_run").get() == 0); REQUIRE(E_x_1_fromRun1.getAttribute("updated_in_run").get() == 1); REQUIRE(E_y_0_fromRun1.getAttribute("updated_in_run").get() == 1); + REQUIRE( + E_y_0_fromRun1.getAttribute("changed_datatype_in_run") + .get() == "one"); auto chunk_E_x_0_fromRun0 = E_x_0_fromRun0.loadChunk({0}, {1}); auto chunk_E_x_1_fromRun1 = E_x_1_fromRun1.loadChunk({0}, {1}); @@ -7516,6 +7523,7 @@ void groupbased_read_write(std::string const &ext) REQUIRE(*chunk_E_x_0_fromRun0 == 0); REQUIRE(*chunk_E_x_1_fromRun1 == 1); REQUIRE(*chunk_E_y_0_fromRun1 == 1); + read.close(); } // check that truncation works correctly @@ -7529,12 +7537,14 @@ void groupbased_read_write(std::string const &ext) E_x.storeChunkRaw(&data, {0}, {1}); E_x.setAttribute("updated_in_run", 2); + write.close(); } { Series read(filename, Access::READ_ONLY); REQUIRE(read.iterations.size() == 1); REQUIRE(read.iterations.count(2) == 1); + read.close(); } } diff --git a/test/python/unittest/Test.py b/test/python/unittest/Test.py index f094b46a2b..081d9eabf7 100644 --- a/test/python/unittest/Test.py +++ b/test/python/unittest/Test.py @@ -16,8 +16,8 @@ # Define the test suite. def suite(): suites = [ - unittest.makeSuite(APITest), - ] + unittest.TestLoader().loadTestsFromTestCase(testCaseClass=APITest), + ] return unittest.TestSuite(suites)