From 58028ea0ee8bd91099da0d85654cd82f0ad6ac8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 8 Aug 2023 19:35:34 +0200 Subject: [PATCH] ADIOS2 schema 2022_07_26, based on ADIOS2 modifiable attributes (#1310) * Deactivate 2021 ADIOS2 schema * Move ADIOS2FilePosition::GD to GroupOrDataset * Fix changesOverSteps parameter * Reimplement CurrentStep() for engines without support * Main implementation: Third schema in ADIOS2 * Remove old schema from tests and use new schema in tests * Ensure support for changing constant record components * Update CI Use apple-clang for testing the new layout * Turn this into a feature of schema 0 * Documentation * Mark all parent paths, too * Add test for group_table option * Update documentation docs/source/backends/adios2.rst * Move ADIOS2 version macros to their own header --- .github/workflows/linux.yml | 8 +- .github/workflows/macos.yml | 3 + CMakeLists.txt | 1 - docs/source/backends/adios2.rst | 30 +- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 53 +- .../openPMD/IO/ADIOS/ADIOS2FilePosition.hpp | 15 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 286 +--- .../IO/ADIOS/ADIOS2PreloadAttributes.hpp | 170 --- include/openPMD/IO/ADIOS/macros.hpp | 27 + include/openPMD/IO/IOTask.hpp | 8 +- include/openPMD/backend/Writable.hpp | 8 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 1219 ++++++----------- src/IO/ADIOS/ADIOS2PreloadAttributes.cpp | 282 ---- src/IO/HDF5/HDF5IOHandler.cpp | 3 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 3 +- src/Iteration.cpp | 25 +- src/RecordComponent.cpp | 19 + src/Series.cpp | 3 +- test/ParallelIOTest.cpp | 23 +- test/SerialIOTest.cpp | 357 ++--- 20 files changed, 851 insertions(+), 1692 deletions(-) delete mode 100644 include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp create mode 100644 include/openPMD/IO/ADIOS/macros.hpp delete mode 100644 src/IO/ADIOS/ADIOS2PreloadAttributes.cpp diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 91f089e03a..af47fd6380 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -80,7 +80,7 @@ jobs: find . -name *.bp.dir | xargs -n1 -P1 -I {} rm -rf {} ctest --test-dir build --output-on-failure - clang7_nopy_ompi_h5_ad2_newLayout: + clang7_nopy_ompi_h5_ad2: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false steps: @@ -94,7 +94,7 @@ jobs: sudo apt-get install clang-7 gfortran libopenmpi-dev python3 sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror, OPENPMD2_ADIOS2_SCHEMA: 20210209} + env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror} run: | eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/) spack install @@ -148,7 +148,7 @@ jobs: ctest --test-dir build --output-on-failure # ADIOS2 v2.7.1 - clang8_py38_mpich_h5_ad2_newLayout: + clang8_py38_mpich_h5_ad2: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false steps: @@ -162,7 +162,7 @@ jobs: sudo apt-get install clang-8 gfortran libmpich-dev python3 sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-8, CXX: clang++-8, CXXFLAGS: -Werror, OPENPMD2_ADIOS2_SCHEMA: 20210209} + env: {CC: clang-8, CXX: clang++-8, CXXFLAGS: -Werror} run: | cmake --version mpiexec --version diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 287bbd8ca4..afd7e3b4bc 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -70,6 +70,9 @@ jobs: -DPython_EXECUTABLE=$(which python3) cmake --build build --parallel 3 ctest --test-dir build --verbose + rm -rf build/samples + share/openPMD/download_samples.sh build + OPENPMD2_ADIOS2_USE_GROUP_TABLE=1 ctest --test-dir build --verbose # TODO: apple_conda_ompi_all (similar to conda_ompi_all on Linux) # both OpenMPI and MPICH cause startup (MPI_Init) issues on GitHub Actions diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ebb25fafa..dbcf2db7dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -478,7 +478,6 @@ set(IO_SOURCE src/IO/JSON/JSONFilePosition.cpp src/IO/ADIOS/ADIOS2IOHandler.cpp src/IO/ADIOS/ADIOS2Auxiliary.cpp - src/IO/ADIOS/ADIOS2PreloadAttributes.cpp src/IO/InvalidatableFile.cpp) # library diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst index c49def1fe7..de6b47aaad 100644 --- a/docs/source/backends/adios2.rst +++ b/docs/source/backends/adios2.rst @@ -82,7 +82,7 @@ environment variable default description ``OPENPMD_ADIOS2_HAVE_METADATA_FILE`` ``1`` Online creation of the adios journal file (``1``: yes, ``0``: no). ``OPENPMD_ADIOS2_NUM_SUBSTREAMS`` ``0`` Number of files to be created, 0 indicates maximum number possible. ``OPENPMD_ADIOS2_ENGINE`` ``File`` `ADIOS2 engine `_ -``OPENPMD2_ADIOS2_SCHEMA`` ``0`` ADIOS2 schema version (see below) +``OPENPMD2_ADIOS2_USE_GROUP_TABLE`` ``0`` Use group table (see below) ``OPENPMD_ADIOS2_STATS_LEVEL`` ``0`` whether to generate statistics for variables in ADIOS2. (``1``: yes, ``0``: no). ``OPENPMD_ADIOS2_ASYNC_WRITE`` ``0`` ADIOS2 BP5 engine: 1 means setting "AsyncWrite" in ADIOS2 to "on". Flushes will go to the buffer by default (see ``preferred_flush_target``). ``OPENPMD_ADIOS2_BP5_BufferChunkMB`` ``0`` ADIOS2 BP5 engine: applies when using either EveryoneWrites or EveryoneWritesSerial aggregation @@ -135,14 +135,32 @@ The preferred backend usually depends on the system's native software stack. For fine-tuning at extreme scale or for exotic systems, please refer to the ADIOS2 manual and talk to your filesystem admins and the ADIOS2 authors. Be aware that extreme-scale I/O is a research topic after all. -Experimental new ADIOS2 schema ------------------------------- +Experimental group table feature +-------------------------------- -The experimental new ADIOS2 schema is deprecated and **will be removed soon**. It used to be activated via the JSON parameter ``adios2.schema = 20210209`` or via the environment variable ``export OPENPMD2_ADIOS2_SCHEMA=20210209``. +We are experimenting with a feature that will make the structure of an ADIOS2 file more explicit. +Currently, the hierarchical structure of an openPMD dataset in ADIOS2 is recovered implicitly by inspecting variables and attributes found in the ADIOS2 file. +Inspecting attributes is necessary since not every openPMD group necessarily contains an (array) dataset. +The downside of this approach is that ADIOS2 attributes do not properly interact with ADIOS2 steps, resulting in many problems and workarounds when parsing an ADIOS2 dataset. +An attribute, once defined, cannot be deleted, implying that the ADIOS2 backend will recover groups that might not actually be logically present in the current step. -**Do no longer use these options**, the created datasets will no longer be read by the openPMD-api. +As a result of this behavior, support for ADIOS2 steps is currently restricted. -An alternative data layout with less intrusive changes and similar features is `currently in development `__. +For full support of ADIOS2 steps, we introduce a group table that makes use of modifiable attributes in ADIOS2 v2.9, i.e. attributes that can have different values across steps. + +An openPMD group ```` is present if: + +1. The integer attribute ``__openPMD_groups/`` exists +2. and: + + a. the file is either accessed in random-access mode + b. or the current value of said attribute is equivalent to the current step index. + +This feature can be activated via the JSON/TOML key ``adios2.use_group_table = true`` or via the environment variable ``OPENPMD2_ADIOS2_USE_GROUP_TABLE=1``. +It is fully backward-compatible with the old layout of openPMD in ADIOS2 and mostly forward-compatible (except the support for steps). + +The variable-based encoding of openPMD automatically activates the group table feature. +The group table feature automatically activates the use of ADIOS2 steps (which until version 0.15 was an opt-in feature via ``adios2.engine.usesteps = true``). Memory usage ------------ diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 87a06f1e27..97a3f5539a 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -21,7 +21,10 @@ #pragma once +#include "openPMD/Error.hpp" +#include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/config.hpp" + #if openPMD_HAVE_ADIOS2 #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" @@ -33,9 +36,17 @@ #include #include #include +#endif namespace openPMD { +enum class GroupOrDataset +{ + GROUP, + DATASET +}; + +#if openPMD_HAVE_ADIOS2 namespace detail { // ADIOS2 does not natively support boolean values @@ -119,6 +130,45 @@ namespace detail std::string const &attributeName, bool verbose, VariableOrAttribute voa = VariableOrAttribute::Attribute); + + inline bool readOnly(adios2::Mode mode) + { + switch (mode) + { + case adios2::Mode::Append: + case adios2::Mode::Write: + return false; + case adios2::Mode::Read: +#if openPMD_HAS_ADIOS_2_8 + case adios2::Mode::ReadRandomAccess: +#endif + return true; + case adios2::Mode::Undefined: + case adios2::Mode::Sync: + case adios2::Mode::Deferred: + break; + } + throw error::Internal("Control flow error: No ADIOS2 open mode."); + } + inline bool writeOnly(adios2::Mode mode) + { + switch (mode) + { + case adios2::Mode::Append: + case adios2::Mode::Write: + return true; + case adios2::Mode::Read: +#if openPMD_HAS_ADIOS_2_8 + case adios2::Mode::ReadRandomAccess: +#endif + return false; + case adios2::Mode::Undefined: + case adios2::Mode::Sync: + case adios2::Mode::Deferred: + break; + } + throw error::Internal("Control flow error: No ADIOS2 open mode."); + } } // namespace detail /** @@ -277,6 +327,5 @@ auto switchAdios2VariableType(Datatype dt, Args &&...args) std::to_string(static_cast(dt))); } } -} // namespace openPMD - #endif // openPMD_HAVE_ADIOS2 +} // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ADIOS2FilePosition.hpp b/include/openPMD/IO/ADIOS/ADIOS2FilePosition.hpp index 7203c33077..399c1f0375 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2FilePosition.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2FilePosition.hpp @@ -20,6 +20,7 @@ */ #pragma once +#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/AbstractFilePosition.hpp" #include #include @@ -28,27 +29,21 @@ namespace openPMD { struct ADIOS2FilePosition : public AbstractFilePosition { - enum class GD - { - GROUP, - DATASET - }; - - ADIOS2FilePosition(std::string s, GD groupOrDataset) + ADIOS2FilePosition(std::string s, GroupOrDataset groupOrDataset) : location{std::move(s)}, gd{groupOrDataset} {} - explicit ADIOS2FilePosition(GD groupOrDataset) + explicit ADIOS2FilePosition(GroupOrDataset groupOrDataset) : ADIOS2FilePosition{"/", groupOrDataset} {} - ADIOS2FilePosition() : ADIOS2FilePosition{GD::GROUP} + ADIOS2FilePosition() : ADIOS2FilePosition{GroupOrDataset::GROUP} {} /** * Convention: Starts with slash '/', ends without. */ std::string location; - GD gd; + GroupOrDataset gd; }; // ADIOS2FilePosition } // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 53a46282a3..d053043243 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -22,7 +22,6 @@ #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" -#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" @@ -68,8 +67,8 @@ namespace detail struct DatasetReader; struct AttributeReader; struct AttributeWriter; - struct OldAttributeReader; - struct OldAttributeWriter; + struct AttributeReader; + struct AttributeWriter; template struct AttributeTypes; struct DatasetOpener; @@ -85,25 +84,11 @@ namespace detail struct RunUniquePtrPut; } // namespace detail -namespace ADIOS2Schema +enum class UseGroupTable { - using schema_t = uint64_t; - /* - * Original ADIOS schema. - */ - constexpr schema_t schema_0000_00_00 = 00000000; - /* - * This introduces attribute layout via scalar ADIOS variables. - */ - constexpr schema_t schema_2021_02_09 = 20210209; - - enum class SupportedSchema : char - { - s_0000_00_00, - s_2021_02_09 - }; -} // namespace ADIOS2Schema -using SupportedSchema = ADIOS2Schema::SupportedSchema; + Yes, + No +}; class ADIOS2IOHandlerImpl : public AbstractIOHandlerImplCommon @@ -114,8 +99,8 @@ class ADIOS2IOHandlerImpl friend struct detail::DatasetReader; friend struct detail::AttributeReader; friend struct detail::AttributeWriter; - friend struct detail::OldAttributeReader; - friend struct detail::OldAttributeWriter; + friend struct detail::AttributeReader; + friend struct detail::AttributeWriter; template friend struct detail::AttributeTypes; friend struct detail::DatasetOpener; @@ -241,8 +226,9 @@ class ADIOS2IOHandlerImpl std::optional m_communicator; #endif /* - * If the iteration encoding is variableBased, we default to using the - * 2021_02_09 schema since it allows mutable attributes. + * If the iteration encoding is variableBased, we default to using a group + * table, since it is the only reliable way to recover currently active + * groups. */ IterationEncoding m_iterationEncoding = IterationEncoding::groupBased; /** @@ -255,9 +241,10 @@ class ADIOS2IOHandlerImpl std::string m_userSpecifiedExtension; /* - * Empty option: No schema has been explicitly selected, use default. + * Empty option: No choice about the group table has been explicitly made, + * use default. */ - std::optional m_schema; + std::optional m_useGroupTable; enum class UseSpan : char { @@ -268,41 +255,23 @@ class ADIOS2IOHandlerImpl UseSpan m_useSpanBasedPutByDefault = UseSpan::Auto; - enum class AttributeLayout : char + enum class ModifiableAttributes : char { - ByAdiosAttributes, - ByAdiosVariables + Yes, + No, + Unspecified }; - inline SupportedSchema schema() const - { - if (!m_schema.has_value()) - { - return SupportedSchema::s_0000_00_00; - } - switch (m_schema.value()) - { - case ADIOS2Schema::schema_0000_00_00: - return SupportedSchema::s_0000_00_00; - case ADIOS2Schema::schema_2021_02_09: - return SupportedSchema::s_2021_02_09; - default: - throw std::runtime_error( - "[ADIOS2] Encountered unsupported schema version: " + - std::to_string(m_schema.value())); - } - } + ModifiableAttributes m_modifiableAttributes = + ModifiableAttributes::Unspecified; - inline AttributeLayout attributeLayout() const + inline UseGroupTable useGroupTable() const { - switch (schema()) + if (!m_useGroupTable.has_value()) { - case SupportedSchema::s_0000_00_00: - return AttributeLayout::ByAdiosAttributes; - case SupportedSchema::s_2021_02_09: - return AttributeLayout::ByAdiosVariables; + return UseGroupTable::No; } - throw std::runtime_error("Unreachable!"); + return m_useGroupTable.value(); } struct ParameterizedOperator @@ -414,7 +383,7 @@ class ADIOS2IOHandlerImpl * Figure out whether the Writable corresponds with a * group or a dataset. */ - ADIOS2FilePosition::GD groupOrDataset(Writable *); + GroupOrDataset groupOrDataset(Writable *); enum class IfFileNotOpen : bool { @@ -457,9 +426,8 @@ namespace ADIOS2Defaults constexpr const_str str_usesstepsAttribute = "__openPMD_internal/useSteps"; constexpr const_str str_adios2Schema = "__openPMD_internal/openPMD2_adios2_schema"; - constexpr const_str str_isBooleanOldLayout = "__is_boolean__"; - constexpr const_str str_isBooleanNewLayout = - "__openPMD_internal/is_boolean"; + constexpr const_str str_isBoolean = "__is_boolean__"; + constexpr const_str str_activeTablePrefix = "__openPMD_groups"; } // namespace ADIOS2Defaults namespace detail @@ -484,10 +452,11 @@ namespace detail static constexpr char const *errorMsg = "ADIOS2: readDataset()"; }; - struct OldAttributeReader + struct AttributeReader { template static Datatype call( + ADIOS2IOHandlerImpl &, adios2::IO &IO, std::string name, std::shared_ptr resource); @@ -496,7 +465,7 @@ namespace detail static Datatype call(Params &&...); }; - struct OldAttributeWriter + struct AttributeWriter { template static void call( @@ -508,29 +477,6 @@ namespace detail static void call(Params &&...); }; - struct AttributeReader - { - template - static Datatype call( - adios2::IO &IO, - detail::PreloadAdiosAttributes const &preloadedAttributes, - std::string name, - std::shared_ptr resource); - - template - static Datatype call(Params &&...); - }; - - struct AttributeWriter - { - template - static void - call(detail::BufferedAttributeWrite ¶ms, BufferedActions &fileData); - - template - static void call(Params &&...); - }; - struct DatasetOpener { template @@ -608,17 +554,6 @@ namespace detail template struct AttributeTypes { - static void createAttribute( - adios2::IO &IO, - adios2::Engine &engine, - detail::BufferedAttributeWrite ¶ms, - T value); - - static Datatype readAttribute( - detail::PreloadAdiosAttributes const &, - std::string name, - std::shared_ptr resource); - /** * @brief Is the attribute given by parameters name and val already * defined exactly in that way within the given IO? @@ -642,27 +577,6 @@ namespace detail template <> struct AttributeTypes> { - static void createAttribute( - adios2::IO &, - adios2::Engine &, - detail::BufferedAttributeWrite &, - std::complex) - { - throw std::runtime_error( - "[ADIOS2] Internal error: no support for long double complex " - "attribute types"); - } - - static Datatype readAttribute( - detail::PreloadAdiosAttributes const &, - std::string, - std::shared_ptr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: no support for long double complex " - "attribute types"); - } - static bool attributeUnchanged(adios2::IO &, std::string, std::complex) { @@ -675,27 +589,6 @@ namespace detail template <> struct AttributeTypes>> { - static void createAttribute( - adios2::IO &, - adios2::Engine &, - detail::BufferedAttributeWrite &, - const std::vector> &) - { - throw std::runtime_error( - "[ADIOS2] Internal error: no support for long double complex " - "vector attribute types"); - } - - static Datatype readAttribute( - detail::PreloadAdiosAttributes const &, - std::string, - std::shared_ptr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: no support for long double complex " - "vector attribute types"); - } - static bool attributeUnchanged( adios2::IO &, std::string, std::vector>) { @@ -708,17 +601,6 @@ namespace detail template struct AttributeTypes> { - static void createAttribute( - adios2::IO &IO, - adios2::Engine &engine, - detail::BufferedAttributeWrite ¶ms, - const std::vector &value); - - static Datatype readAttribute( - detail::PreloadAdiosAttributes const &, - std::string name, - std::shared_ptr resource); - static bool attributeUnchanged(adios2::IO &IO, std::string name, std::vector val) { @@ -746,17 +628,6 @@ namespace detail template <> struct AttributeTypes> { - static void createAttribute( - adios2::IO &IO, - adios2::Engine &engine, - detail::BufferedAttributeWrite ¶ms, - const std::vector &vec); - - static Datatype readAttribute( - detail::PreloadAdiosAttributes const &, - std::string name, - std::shared_ptr resource); - static bool attributeUnchanged( adios2::IO &IO, std::string name, std::vector val) { @@ -784,17 +655,6 @@ namespace detail template struct AttributeTypes> { - static void createAttribute( - adios2::IO &IO, - adios2::Engine &engine, - detail::BufferedAttributeWrite ¶ms, - const std::array &value); - - static Datatype readAttribute( - detail::PreloadAdiosAttributes const &, - std::string name, - std::shared_ptr resource); - static bool attributeUnchanged( adios2::IO &IO, std::string name, std::array val) { @@ -849,17 +709,6 @@ namespace detail return r != 0; } - static void createAttribute( - adios2::IO &IO, - adios2::Engine &engine, - detail::BufferedAttributeWrite ¶ms, - bool value); - - static Datatype readAttribute( - detail::PreloadAdiosAttributes const &, - std::string name, - std::shared_ptr resource); - static bool attributeUnchanged(adios2::IO &IO, std::string name, bool val) { @@ -925,32 +774,6 @@ namespace detail void run(BufferedActions &); }; - struct OldBufferedAttributeRead : BufferedAction - { - Parameter param; - std::string name; - - void run(BufferedActions &) override; - }; - - struct BufferedAttributeRead - { - Parameter param; - std::string name; - - void run(BufferedActions &); - }; - - struct BufferedAttributeWrite : BufferedAction - { - std::string name; - Datatype dtype; - Attribute::resource resource; - std::vector bufferForVecString; - - void run(BufferedActions &) override; - }; - struct I_UpdateSpan { virtual void *update() = 0; @@ -1017,23 +840,6 @@ namespace detail * Drained upon BufferedActions::flush(). */ std::vector> m_buffer; - /** - * Buffer for attributes to be written in the new (variable-based) - * attribute layout. - * Reason: If writing one variable twice within the same ADIOS step, - * it is undefined which value ADIOS2 will store. - * We want the last write operation to succeed, so this map stores - * attribute writes by attribute name, allowing us to override older - * write commands. - * The queue is drained only when closing a step / the engine. - */ - std::map m_attributeWrites; - /** - * @todo This one is unnecessary, in the new schema, attribute reads do - * not need to be deferred, but can happen instantly without performance - * penalty, once preloadAttributes has been filled. - */ - std::vector m_attributeReads; /** * When receiving a unique_ptr, we know that the buffer is ours and * ours alone. So, for performance reasons, show the buffer to ADIOS2 as @@ -1058,7 +864,6 @@ namespace detail * This map is cleared upon flush points. */ std::map> m_updateSpans; - PreloadAdiosAttributes preloadAttributes; /* * We call an attribute committed if the step during which it was @@ -1095,6 +900,8 @@ namespace detail */ void finalize(); + UseGroupTable detectGroupTable(); + adios2::Engine &getEngine(); adios2::Engine &requireActiveStep(); @@ -1134,10 +941,9 @@ namespace detail * * adios2::Engine::EndStep * * adios2::Engine::Perform(Puts|Gets) * * adios2::Engine::Close - * @param writeLatePuts Some things are deferred until right before + * @param writeLatePuts Deferred until right before * Engine::EndStep() or Engine::Close(): - * 1) Writing attributes in new ADIOS2 schema. - * 2) Running unique_ptr Put()s. + * Running unique_ptr Put()s. * @param flushUnconditionally Whether to run the functor even if no * deferred IO tasks had been queued. */ @@ -1194,6 +1000,10 @@ namespace detail */ void invalidateVariablesMap(); + void markActive(Writable *); + + // bool isActive(std::string const & path); + /* * streamStatus is NoStream for file-based ADIOS engines. * This is relevant for the method BufferedActions::requireActiveStep, @@ -1285,6 +1095,8 @@ namespace detail }; StreamStatus streamStatus = StreamStatus::OutsideOfStep; + size_t currentStep(); + private: ADIOS2IOHandlerImpl *m_impl; std::optional m_engine; //! ADIOS engine @@ -1293,6 +1105,12 @@ namespace detail */ std::string m_engineType; + /* + * Not all engines support the CurrentStep() call, so we have to + * implement this manually. + */ + size_t m_currentStep = 0; + /* * ADIOS2 does not give direct access to its internal attribute and * variable maps, but will instead give access to copies of them. @@ -1308,6 +1126,8 @@ namespace detail std::optional m_availableAttributes; std::optional m_availableVariables; + std::set m_pathsMarkedAsActive; + /* * Cannot write attributes right after opening the engine * https://github.com/ornladios/ADIOS2/issues/3433 @@ -1318,9 +1138,9 @@ namespace detail */ bool finalized = false; - inline SupportedSchema schema() const + [[nodiscard]] inline UseGroupTable useGroupTable() const { - return m_impl->schema(); + return m_impl->useGroupTable(); } void create_IO(); @@ -1328,12 +1148,6 @@ namespace detail void configure_IO(ADIOS2IOHandlerImpl &impl); void configure_IO_Read(std::optional userSpecifiedUsesteps); void configure_IO_Write(std::optional userSpecifiedUsesteps); - - using AttributeLayout = ADIOS2IOHandlerImpl::AttributeLayout; - inline AttributeLayout attributeLayout() const - { - return m_impl->attributeLayout(); - } }; } // namespace detail diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp deleted file mode 100644 index c2d9da0cbc..0000000000 --- a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp +++ /dev/null @@ -1,170 +0,0 @@ -/* Copyright 2020-2021 Franz Poeschel - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ -#pragma once - -#include "openPMD/config.hpp" -#if openPMD_HAVE_ADIOS2 - -#include -#include -#include -#include -#include -#include - -#include "openPMD/Datatype.hpp" -#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" - -namespace openPMD -{ -namespace detail -{ - /** - * @brief Pointer to an attribute's data along with its shape. - * - * @tparam T Underlying attribute data type. - */ - template - struct AttributeWithShape - { - adios2::Dims shape; - T const *data; - }; - - /** - * Class that is responsible for scheduling and buffering openPMD attribute - * loads from ADIOS2, if using ADIOS variables to store openPMD attributes. - * - * Reasoning: ADIOS variables can be of any shape and size, and ADIOS cannot - * know which variables to buffer. While it will preload and buffer scalar - * variables, openPMD also stores vector-type attributes which are not - * preloaded. Since in Streaming setups, every variable load requires full - * communication back to the writer, this can quickly become very expensive. - * Hence, do this manually. - * - */ - class PreloadAdiosAttributes - { - public: - /** - * Internally used struct to store meta information on a buffered - * attribute. Public for simplicity (helper struct in the - * implementation uses it). - */ - struct AttributeLocation - { - adios2::Dims shape; - size_t offset; - Datatype dt; - char *destroy = nullptr; - - AttributeLocation() = delete; - AttributeLocation(adios2::Dims shape, size_t offset, Datatype dt); - - AttributeLocation(AttributeLocation const &other) = delete; - AttributeLocation & - operator=(AttributeLocation const &other) = delete; - - AttributeLocation(AttributeLocation &&other); - AttributeLocation &operator=(AttributeLocation &&other); - - ~AttributeLocation(); - }; - - private: - /* - * Allocate one large buffer instead of hundreds of single heap - * allocations. - * This will comply with alignment requirements, since - * std::allocator::allocate() will call the untyped new operator - * ::operator new(std::size_t) - * https://en.cppreference.com/w/cpp/memory/allocator/allocate - */ - std::vector m_rawBuffer; - std::map m_offsets; - - public: - explicit PreloadAdiosAttributes() = default; - PreloadAdiosAttributes(PreloadAdiosAttributes const &other) = delete; - PreloadAdiosAttributes & - operator=(PreloadAdiosAttributes const &other) = delete; - - PreloadAdiosAttributes(PreloadAdiosAttributes &&other) = default; - PreloadAdiosAttributes & - operator=(PreloadAdiosAttributes &&other) = default; - - /** - * @brief Schedule attributes for preloading. - * - * This will invalidate all previously buffered attributes. - * This will *not* flush the scheduled loads. This way, attributes can - * be loaded along with the next adios2::Engine flush. - * - * @param IO - * @param engine - */ - void preloadAttributes(adios2::IO &IO, adios2::Engine &engine); - - /** - * @brief Get an attribute that has been buffered previously. - * - * @tparam T The underlying primitive datatype of the attribute. - * Will fail if the type found in ADIOS does not match. - * @param name The full name of the attribute. - * @return Pointer to the buffered attribute along with information on - * the attribute's shape. Valid only until any non-const member - * of PreloadAdiosAttributes is called. - */ - template - AttributeWithShape getAttribute(std::string const &name) const; - - Datatype attributeType(std::string const &name) const; - }; - - template - AttributeWithShape - PreloadAdiosAttributes::getAttribute(std::string const &name) const - { - auto it = m_offsets.find(name); - if (it == m_offsets.end()) - { - throw std::runtime_error( - "[ADIOS2] Requested attribute not found: " + name); - } - AttributeLocation const &location = it->second; - Datatype determinedDatatype = determineDatatype(); - if (location.dt != determinedDatatype) - { - std::stringstream errorMsg; - errorMsg << "[ADIOS2] Wrong datatype for attribute: " << name - << "(location.dt=" << location.dt - << ", T=" << determineDatatype() << ")"; - throw std::runtime_error(errorMsg.str()); - } - AttributeWithShape res; - res.shape = location.shape; - res.data = reinterpret_cast(&m_rawBuffer[location.offset]); - return res; - } -} // namespace detail -} // namespace openPMD - -#endif // openPMD_HAVE_ADIOS2 diff --git a/include/openPMD/IO/ADIOS/macros.hpp b/include/openPMD/IO/ADIOS/macros.hpp new file mode 100644 index 0000000000..59b630a112 --- /dev/null +++ b/include/openPMD/IO/ADIOS/macros.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include "openPMD/config.hpp" + +#if openPMD_HAVE_ADIOS2 + +#include + +/* + * ADIOS2 v2.8 brings mode::ReadRandomAccess + */ +#define openPMD_HAS_ADIOS_2_8 \ + (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208) +/* + * ADIOS2 v2.9 brings modifiable attributes (technically already in v2.8, but + * there are too many bugs, so we only support it beginning with v2.9). + * Group table feature requires ADIOS2 v2.9. + */ +#define openPMD_HAS_ADIOS_2_9 \ + (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209) + +#else + +#define openPMD_HAS_ADIOS_2_8 0 +#define openPMD_HAS_ADIOS_2_9 0 + +#endif diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 0ee6cd66d7..5a5c4fc54e 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -561,7 +561,13 @@ struct OPENPMDAPI_EXPORT Parameter * otherwise writing should be skipped. * The frontend is responsible for handling both situations. */ - bool changesOverSteps = false; + enum class ChangesOverSteps + { + No, + Yes, + IfPossible + }; + ChangesOverSteps changesOverSteps = ChangesOverSteps::No; Attribute::resource resource; }; diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index c585893fc3..7aeead3dfe 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -50,6 +50,10 @@ namespace internal class AttributableData; class SeriesData; } // namespace internal +namespace detail +{ + struct BufferedActions; +} /** @brief Layer to mirror structure of logical data and persistent data in * file. @@ -79,9 +83,11 @@ class Writable final friend class Record; friend class AbstractIOHandlerImpl; friend class ADIOS2IOHandlerImpl; + friend struct detail::BufferedActions; friend class HDF5IOHandlerImpl; friend class ParallelHDF5IOHandlerImpl; - friend class AbstractIOHandlerImplCommon; + template + friend class AbstractIOHandlerImplCommon; friend class JSONIOHandlerImpl; friend struct test::TestHelper; friend std::string concrete_h5_file_position(Writable *); diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index ab063b5a91..962585d1d9 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -23,6 +23,7 @@ #include "openPMD/Datatype.hpp" #include "openPMD/Error.hpp" +#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -64,8 +65,6 @@ namespace openPMD #if openPMD_HAVE_ADIOS2 -#define HAS_ADIOS_2_8 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208) - #if openPMD_HAVE_MPI ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( @@ -148,19 +147,23 @@ void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) [](unsigned char c) { return std::tolower(c); }); // environment-variable based configuration - if (int schemaViaEnv = auxiliary::getEnvNum("OPENPMD2_ADIOS2_SCHEMA", -1); - schemaViaEnv != -1) + if (int groupTableViaEnv = + auxiliary::getEnvNum("OPENPMD2_ADIOS2_USE_GROUP_TABLE", -1); + groupTableViaEnv != -1) { - m_schema = schemaViaEnv; + m_useGroupTable = + groupTableViaEnv == 0 ? UseGroupTable::No : UseGroupTable::Yes; } if (cfg.json().contains("adios2")) { m_config = cfg["adios2"]; - if (m_config.json().contains("schema")) + if (m_config.json().contains("use_group_table")) { - m_schema = m_config["schema"].json().get(); + m_useGroupTable = m_config["use_group_table"].json().get() + ? UseGroupTable::Yes + : UseGroupTable::No; } if (m_config.json().contains("use_span_based_put")) @@ -170,6 +173,14 @@ void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) : UseSpan::No; } + if (m_config.json().contains("modifiable_attributes")) + { + m_modifiableAttributes = + m_config["modifiable_attributes"].json().get() + ? ModifiableAttributes::Yes + : ModifiableAttributes::No; + } + auto engineConfig = config(ADIOS2Defaults::str_engine); if (!engineConfig.json().is_null()) { @@ -199,6 +210,21 @@ void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) defaultOperators = std::move(operators.value()); } } +#if !openPMD_HAS_ADIOS_2_9 + if (m_modifiableAttributes == ModifiableAttributes::Yes) + { + throw error::OperationUnsupportedInBackend( + m_handler->backendName(), + "Modifiable attributes require ADIOS2 >= v2.9."); + } + if (m_useGroupTable.has_value() && + m_useGroupTable.value() == UseGroupTable::Yes) + { + throw error::OperationUnsupportedInBackend( + m_handler->backendName(), + "ADIOS2 group table feature requires ADIOS2 >= v2.9."); + } +#endif } std::optional> @@ -578,7 +604,7 @@ void ADIOS2IOHandlerImpl::createPath( Writable *writable, const Parameter ¶meters) { std::string path; - refreshFileFromParent(writable, /* preferParentFile = */ true); + auto file = refreshFileFromParent(writable, /* preferParentFile = */ true); /* Sanitize path */ if (!auxiliary::starts_with(parameters.path, '/')) @@ -595,8 +621,17 @@ void ADIOS2IOHandlerImpl::createPath( * They are implicitly created with the paths of variables/attributes. */ writable->written = true; - writable->abstractFilePosition = std::make_shared( - path, ADIOS2FilePosition::GD::GROUP); + writable->abstractFilePosition = + std::make_shared(path, GroupOrDataset::GROUP); + + switch (useGroupTable()) + { + case UseGroupTable::No: + break; + case UseGroupTable::Yes: + getFileData(file, IfFileNotOpen::ThrowError).markActive(writable); + break; + } } void ADIOS2IOHandlerImpl::createDataset( @@ -616,7 +651,7 @@ void ADIOS2IOHandlerImpl::createDataset( auto const file = refreshFileFromParent(writable, /* preferParentFile = */ true); auto filePos = setAndGetFilePosition(writable, name); - filePos->gd = ADIOS2FilePosition::GD::DATASET; + filePos->gd = GroupOrDataset::DATASET; auto const varName = nameOfVariable(writable); std::vector operators; @@ -757,7 +792,7 @@ void ADIOS2IOHandlerImpl::openPath( Writable *writable, const Parameter ¶meters) { /* Sanitize path */ - refreshFileFromParent(writable, /* preferParentFile = */ true); + auto file = refreshFileFromParent(writable, /* preferParentFile = */ true); std::string prefix = filePositionToString(setAndGetFilePosition(writable->parent)); std::string suffix = auxiliary::removeSlashes(parameters.path); @@ -768,8 +803,17 @@ void ADIOS2IOHandlerImpl::openPath( * They are implicitly created with the paths of variables/attributes. */ writable->abstractFilePosition = std::make_shared( - prefix + infix + suffix, ADIOS2FilePosition::GD::GROUP); + prefix + infix + suffix, GroupOrDataset::GROUP); writable->written = true; + + switch (useGroupTable()) + { + case UseGroupTable::No: + break; + case UseGroupTable::Yes: + getFileData(file, IfFileNotOpen::ThrowError).markActive(writable); + break; + } } void ADIOS2IOHandlerImpl::openDataset( @@ -778,12 +822,12 @@ void ADIOS2IOHandlerImpl::openDataset( auto name = auxiliary::removeSlashes(parameters.name); writable->abstractFilePosition.reset(); auto pos = setAndGetFilePosition(writable, name); - pos->gd = ADIOS2FilePosition::GD::DATASET; + pos->gd = GroupOrDataset::DATASET; auto file = refreshFileFromParent(writable, /* preferParentFile = */ true); auto varName = nameOfVariable(writable); + auto &fileData = getFileData(file, IfFileNotOpen::ThrowError); *parameters.dtype = - detail::fromADIOS2Type(getFileData(file, IfFileNotOpen::ThrowError) - .m_IO.VariableType(varName)); + detail::fromADIOS2Type(fileData.m_IO.VariableType(varName)); switchAdios2VariableType( *parameters.dtype, this, file, varName, parameters); writable->written = true; @@ -835,49 +879,34 @@ void ADIOS2IOHandlerImpl::writeDataset( void ADIOS2IOHandlerImpl::writeAttribute( Writable *writable, const Parameter ¶meters) { - switch (attributeLayout()) +#if openPMD_HAS_ADIOS_2_9 + switch (useGroupTable()) { - case AttributeLayout::ByAdiosAttributes: - if (parameters.changesOverSteps) + case UseGroupTable::No: + if (parameters.changesOverSteps == + Parameter::ChangesOverSteps::Yes) { // cannot do this return; } - switchType( - parameters.dtype, this, writable, parameters); - break; - case AttributeLayout::ByAdiosVariables: { - VERIFY_ALWAYS( - access::write(m_handler->m_backendAccess), - "[ADIOS2] Cannot write attribute in read-only mode."); - auto pos = setAndGetFilePosition(writable); - auto file = - refreshFileFromParent(writable, /* preferParentFile = */ false); - auto fullName = nameOfAttribute(writable, parameters.name); - auto prefix = filePositionToString(pos); - auto &filedata = getFileData(file, IfFileNotOpen::ThrowError); - if (parameters.changesOverSteps && - filedata.streamStatus == - detail::BufferedActions::StreamStatus::NoStream) - { - // cannot do this - return; - } - filedata.requireActiveStep(); - filedata.invalidateAttributesMap(); - m_dirty.emplace(std::move(file)); - - // this intentionally overwrites previous writes - auto &bufferedWrite = filedata.m_attributeWrites[fullName]; - bufferedWrite.name = fullName; - bufferedWrite.dtype = parameters.dtype; - bufferedWrite.resource = parameters.resource; + break; + case UseGroupTable::Yes: { break; } default: throw std::runtime_error("Unreachable!"); } +#else + if (parameters.changesOverSteps == + Parameter::ChangesOverSteps::Yes) + { + // cannot do this + return; + } +#endif + switchType( + parameters.dtype, this, writable, parameters); } void ADIOS2IOHandlerImpl::readDataset( @@ -1033,27 +1062,21 @@ void ADIOS2IOHandlerImpl::readAttribute( auto pos = setAndGetFilePosition(writable); detail::BufferedActions &ba = getFileData(file, IfFileNotOpen::ThrowError); ba.requireActiveStep(); - switch (attributeLayout()) - { - using AL = AttributeLayout; - case AL::ByAdiosAttributes: { - detail::OldBufferedAttributeRead bar; - bar.name = nameOfAttribute(writable, parameters.name); - bar.param = parameters; - ba.enqueue(std::move(bar)); - break; - } - case AL::ByAdiosVariables: { - detail::BufferedAttributeRead bar; - bar.name = nameOfAttribute(writable, parameters.name); - bar.param = parameters; - ba.m_attributeReads.push_back(std::move(bar)); - break; - } - default: - throw std::runtime_error("Unreachable!"); + auto name = nameOfAttribute(writable, parameters.name); + + auto type = detail::attributeInfo(ba.m_IO, name, /* verbose = */ true); + if (type == Datatype::UNDEFINED) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::NotFound, + "ADIOS2", + name); } - m_dirty.emplace(std::move(file)); + + Datatype ret = switchType( + type, *this, ba.m_IO, name, parameters.resource); + *parameters.dtype = ret; } void ADIOS2IOHandlerImpl::listPaths( @@ -1093,45 +1116,9 @@ void ADIOS2IOHandlerImpl::listPaths( */ std::vector delete_me; - switch (attributeLayout()) + switch (useGroupTable()) { - using AL = AttributeLayout; - case AL::ByAdiosVariables: { - std::vector vars = - fileData.availableVariablesPrefixed(myName); - for (auto var : vars) - { - // since current Writable is a group and no dataset, - // var == "__data__" is not possible - if (auxiliary::ends_with(var, "/__data__")) - { - // here be datasets - var = auxiliary::replace_last(var, "/__data__", ""); - auto firstSlash = var.find_first_of('/'); - if (firstSlash != std::string::npos) - { - var = var.substr(0, firstSlash); - subdirs.emplace(std::move(var)); - } - else - { // var is a dataset at the current level - delete_me.push_back(std::move(var)); - } - } - else - { - // here be attributes - auto firstSlash = var.find_first_of('/'); - if (firstSlash != std::string::npos) - { - var = var.substr(0, firstSlash); - subdirs.emplace(std::move(var)); - } - } - } - break; - } - case AL::ByAdiosAttributes: { + case UseGroupTable::No: { std::vector vars = fileData.availableVariablesPrefixed(myName); for (auto var : vars) @@ -1160,6 +1147,56 @@ void ADIOS2IOHandlerImpl::listPaths( } break; } + case UseGroupTable::Yes: { + { + auto tablePrefix = ADIOS2Defaults::str_activeTablePrefix + myName; + std::vector attrs = + fileData.availableAttributesPrefixed(tablePrefix); + if (fileData.streamStatus == + detail::BufferedActions::StreamStatus::DuringStep) + { + auto currentStep = fileData.currentStep(); + for (auto const &attrName : attrs) + { + using table_t = unsigned long long; + auto attr = fileData.m_IO.InquireAttribute( + tablePrefix + attrName); + if (!attr) + { + std::cerr << "[ADIOS2 backend] Unable to inquire group " + "table value for group '" + << myName << attrName + << "', will pretend it does not exist." + << std::endl; + continue; + } + if (attr.Data()[0] != currentStep) + { + // group wasn't defined in current step + continue; + } + auto firstSlash = attrName.find_first_of('/'); + if (firstSlash == std::string::npos) + { + subdirs.emplace(attrName); + } + // else // should we maybe consider deeper groups too? + } + } + else + { + for (auto const &attrName : attrs) + { + auto firstSlash = attrName.find_first_of('/'); + if (firstSlash == std::string::npos) + { + subdirs.emplace(attrName); + } + } + } + } + break; + } } for (auto &d : delete_me) @@ -1199,17 +1236,6 @@ void ADIOS2IOHandlerImpl::listDatasets( std::unordered_set subdirs; for (auto var : fileData.availableVariablesPrefixed(myName)) { - if (attributeLayout() == AttributeLayout::ByAdiosVariables) - { - // since current Writable is a group and no dataset, - // var == "__data__" is not possible - if (!auxiliary::ends_with(var, "/__data__")) - { - continue; - } - // variable is now definitely a dataset, let's strip the suffix - var = auxiliary::replace_last(var, "/__data__", ""); - } // if string still contains a slash, variable is a dataset below the // current group // we only want datasets contained directly within the current group @@ -1243,25 +1269,11 @@ void ADIOS2IOHandlerImpl::listAttributes( auto &ba = getFileData(file, IfFileNotOpen::ThrowError); ba.requireActiveStep(); // make sure that the attributes are present - std::vector attrs; - switch (attributeLayout()) - { - using AL = AttributeLayout; - case AL::ByAdiosAttributes: - attrs = ba.availableAttributesPrefixed(attributePrefix); - break; - case AL::ByAdiosVariables: - attrs = ba.availableVariablesPrefixed(attributePrefix); - break; - } + std::vector attrs = + ba.availableAttributesPrefixed(attributePrefix); + for (auto &rawAttr : attrs) { - if (attributeLayout() == AttributeLayout::ByAdiosVariables && - (auxiliary::ends_with(rawAttr, "/__data__") || - rawAttr == "__data__")) - { - continue; - } auto attr = auxiliary::removeSlashes(rawAttr); if (attr.find_last_of('/') == std::string::npos) { @@ -1342,7 +1354,7 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) { case Access::CREATE: return adios2::Mode::Write; -#if HAS_ADIOS_2_8 +#if openPMD_HAS_ADIOS_2_8 case Access::READ_LINEAR: return adios2::Mode::Read; case Access::READ_ONLY: @@ -1356,7 +1368,7 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) if (auxiliary::directory_exists(fullPath) || auxiliary::file_exists(fullPath)) { -#if HAS_ADIOS_2_8 +#if openPMD_HAS_ADIOS_2_8 return adios2::Mode::ReadRandomAccess; #else return adios2::Mode::Read; @@ -1436,29 +1448,7 @@ ADIOS2IOHandlerImpl::getCompressionOperator(std::string const &compression) std::string ADIOS2IOHandlerImpl::nameOfVariable(Writable *writable) { auto filepos = setAndGetFilePosition(writable); - auto res = filePositionToString(filepos); - if (attributeLayout() == AttributeLayout::ByAdiosAttributes) - { - return res; - } - switch (filepos->gd) - { - case ADIOS2FilePosition::GD::GROUP: - return res; - case ADIOS2FilePosition::GD::DATASET: - if (auxiliary::ends_with(res, '/')) - { - return res + "__data__"; - } - else - { - // By convention, this path should always be taken - // But let's be safe - return res + "/__data__"; - } - default: - throw std::runtime_error("[ADIOS2IOHandlerImpl] Unreachable!"); - } + return filePositionToString(filepos); } std::string @@ -1469,7 +1459,7 @@ ADIOS2IOHandlerImpl::nameOfAttribute(Writable *writable, std::string attribute) extendFilePosition(pos, auxiliary::removeSlashes(attribute))); } -ADIOS2FilePosition::GD ADIOS2IOHandlerImpl::groupOrDataset(Writable *writable) +GroupOrDataset ADIOS2IOHandlerImpl::groupOrDataset(Writable *writable) { return setAndGetFilePosition(writable)->gd; } @@ -1585,11 +1575,13 @@ namespace detail } template - Datatype OldAttributeReader::call( + Datatype AttributeReader::call( + ADIOS2IOHandlerImpl &impl, adios2::IO &IO, std::string name, std::shared_ptr resource) { + (void)impl; /* * If we store an attribute of boolean type, we store an additional * attribute prefixed with '__is_boolean__' to indicate this information @@ -1607,8 +1599,8 @@ namespace detail name + "'."); } - std::string metaAttr = - ADIOS2Defaults::str_isBooleanOldLayout + name; + std::string metaAttr; + metaAttr = ADIOS2Defaults::str_isBoolean + name; /* * In verbose mode, attributeInfo will yield a warning if not * finding the requested attribute. Since we expect the attribute @@ -1617,7 +1609,7 @@ namespace detail */ auto type = attributeInfo( IO, - ADIOS2Defaults::str_isBooleanOldLayout + name, + metaAttr, /* verbose = */ false); if (type == determineDatatype()) @@ -1685,55 +1677,6 @@ namespace detail return determineDatatype(); } - template - Datatype OldAttributeReader::call(Params &&...) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Unknown datatype while " - "trying to read an attribute."); - } - - template - Datatype AttributeReader::call( - adios2::IO &IO, - detail::PreloadAdiosAttributes const &preloadedAttributes, - std::string name, - std::shared_ptr resource) - { - /* - * If we store an attribute of boolean type, we store an additional - * attribute prefixed with '__is_boolean__' to indicate this information - * that would otherwise be lost. Check whether this has been done. - */ - using rep = AttributeTypes::rep; - if constexpr (std::is_same::value) - { - std::string metaAttr = - ADIOS2Defaults::str_isBooleanNewLayout + name; - /* - * In verbose mode, attributeInfo will yield a warning if not - * finding the requested attribute. Since we expect the attribute - * not to be present in many cases (i.e. when it is actually not - * a boolean), let's tell attributeInfo to be quiet. - */ - auto type = attributeInfo( - IO, - ADIOS2Defaults::str_isBooleanNewLayout + name, - /* verbose = */ false); - if (type == determineDatatype()) - { - auto attr = IO.InquireAttribute(metaAttr); - if (attr.Data().size() == 1 && attr.Data()[0] == 1) - { - return AttributeTypes::readAttribute( - preloadedAttributes, name, resource); - } - } - } - return AttributeTypes::readAttribute( - preloadedAttributes, name, resource); - } - template Datatype AttributeReader::call(Params &&...) { @@ -1743,7 +1686,7 @@ namespace detail } template - void OldAttributeWriter::call( + void AttributeWriter::call( ADIOS2IOHandlerImpl *impl, Writable *writable, const Parameter ¶meters) @@ -1764,62 +1707,119 @@ namespace detail adios2::IO IO = filedata.m_IO; impl->m_dirty.emplace(std::move(file)); - std::string t = IO.AttributeType(fullName); - if (!t.empty()) // an attribute is present <=> it has a type +#if openPMD_HAS_ADIOS_2_9 + if (impl->m_modifiableAttributes == + ADIOS2IOHandlerImpl::ModifiableAttributes::No && + parameters.changesOverSteps == + Parameter::ChangesOverSteps::No) +#endif // we only support modifiable attrs for ADIOS2 >= 2.9, so no `if` { - // don't overwrite attributes if they are equivalent - // overwriting is only legal within the same step - - auto attributeModifiable = [&filedata, &fullName]() { - auto it = filedata.uncommittedAttributes.find(fullName); - return it != filedata.uncommittedAttributes.end(); - }; - if (AttributeTypes::attributeUnchanged( - IO, fullName, std::get(parameters.resource))) - { - return; - } - else if (attributeModifiable()) + std::string t = IO.AttributeType(fullName); + if (!t.empty()) // an attribute is present <=> it has a type { - if (detail::fromADIOS2Type(t) != - basicDatatype(determineDatatype())) + // don't overwrite attributes if they are equivalent + // overwriting is only legal within the same step + + auto attributeModifiable = [&filedata, &fullName]() { + auto it = filedata.uncommittedAttributes.find(fullName); + return it != filedata.uncommittedAttributes.end(); + }; + if (AttributeTypes::attributeUnchanged( + IO, fullName, std::get(parameters.resource))) { - if (impl->m_engineType == "bp5") - { - throw error::OperationUnsupportedInBackend( - "ADIOS2", - "Attempting to change datatype of attribute '" + - fullName + - "'. In the BP5 engine, this will lead to " - "corrupted " - "datasets."); - } - else + return; + } + else if (attributeModifiable()) + { + if (detail::fromADIOS2Type(t) != + basicDatatype(determineDatatype())) { - std::cerr << "[ADIOS2] Attempting to change datatype " - "of attribute '" - << fullName - << "'. This invokes undefined behavior. Will " - "proceed." - << std::endl; + if (impl->m_engineType == "bp5") + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", + "Attempting to change datatype of attribute '" + + fullName + + "'. In the BP5 engine, this will lead to " + "corrupted " + "datasets."); + } + else + { + std::cerr + << "[ADIOS2] Attempting to change datatype " + "of attribute '" + << fullName + << "'. This invokes undefined behavior. Will " + "proceed." + << std::endl; + } } + IO.RemoveAttribute(fullName); + } + else + { + std::cerr + << "[Warning][ADIOS2] Cannot modify attribute from " + "previous step: " + << fullName << std::endl; + return; } - IO.RemoveAttribute(fullName); } else { - std::cerr << "[Warning][ADIOS2] Cannot modify attribute from " - "previous step: " - << fullName << std::endl; - return; + filedata.uncommittedAttributes.emplace(fullName); } } - else - { - filedata.uncommittedAttributes.emplace(fullName); - } auto &value = std::get(parameters.resource); +#if openPMD_HAS_ADIOS_2_9 + bool modifiable = impl->m_modifiableAttributes == + ADIOS2IOHandlerImpl::ModifiableAttributes::Yes || + parameters.changesOverSteps != + Parameter::ChangesOverSteps::No; +#else + bool modifiable = impl->m_modifiableAttributes == + ADIOS2IOHandlerImpl::ModifiableAttributes::Yes || + parameters.changesOverSteps == + Parameter::ChangesOverSteps::Yes; +#endif + + auto defineAttribute = + [&IO, &fullName, &modifiable, &impl](auto const &...args) { +#if openPMD_HAS_ADIOS_2_9 + auto attr = IO.DefineAttribute( + fullName, + args..., + /* variableName = */ "", + /* separator = */ "/", + /* allowModification = */ modifiable); +#else + /* + * Defensive coding, normally this condition should be checked + * before getting this far. + */ + if (modifiable) + { + throw error::OperationUnsupportedInBackend( + impl->m_handler->backendName(), + "Modifiable attributes require ADIOS2 >= v2.8."); + } + auto attr = IO.DefineAttribute(fullName, args...); +#endif + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Internal error: Failed defining attribute '" + + fullName + "'."); + } + }; + + /* + * Some old compilers don't like that this is not used in every + * instantiation. + */ + (void)defineAttribute; if constexpr (IsUnsupportedComplex_v) { @@ -1827,72 +1827,23 @@ namespace detail "[ADIOS2] Internal error: no support for long double complex " "attribute types"); } - else if constexpr (auxiliary::IsVector_v) - { - auto attr = - IO.DefineAttribute(fullName, value.data(), value.size()); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed defining attribute '" + - fullName + "'."); - } - } - else if constexpr (auxiliary::IsArray_v) + else if constexpr (auxiliary::IsVector_v || auxiliary::IsArray_v) { - auto attr = - IO.DefineAttribute(fullName, value.data(), value.size()); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed defining attribute '" + - fullName + "'."); - } + defineAttribute(value.data(), value.size()); } else if constexpr (std::is_same_v) { IO.DefineAttribute( - ADIOS2Defaults::str_isBooleanOldLayout + fullName, 1); + ADIOS2Defaults::str_isBoolean + fullName, 1); auto representation = bool_repr::toRep(value); - auto attr = IO.DefineAttribute(fullName, representation); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed defining attribute '" + - fullName + "'."); - } + defineAttribute(representation); } else { - auto attr = IO.DefineAttribute(fullName, value); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed defining attribute '" + - fullName + "'."); - } + defineAttribute(value); } } - template - void OldAttributeWriter::call(Params &&...) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Unknown datatype while " - "trying to write an attribute."); - } - - template - void AttributeWriter::call( - detail::BufferedAttributeWrite ¶ms, BufferedActions &fileData) - { - AttributeTypes::createAttribute( - fileData.m_IO, - fileData.requireActiveStep(), - params, - std::get(params.resource)); - } - template void AttributeWriter::call(Params &&...) { @@ -2100,322 +2051,6 @@ namespace detail // variable has not been found, so we don't fill in any blocks } - template - void AttributeTypes::createAttribute( - adios2::IO &IO, - adios2::Engine &engine, - detail::BufferedAttributeWrite ¶ms, - const T value) - { - auto attr = IO.InquireVariable(params.name); - // @todo check size - if (!attr) - { - // std::cout << "DATATYPE OF " << name << ": " - // << IO.VariableType( name ) << std::endl; - attr = IO.DefineVariable(params.name); - } - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed defining variable '" + - params.name + "'."); - } - engine.Put(attr, value, adios2::Mode::Deferred); - } - - template - Datatype AttributeTypes::readAttribute( - detail::PreloadAdiosAttributes const &preloadedAttributes, - std::string name, - std::shared_ptr resource) - { - detail::AttributeWithShape attr = - preloadedAttributes.getAttribute(name); - if (!(attr.shape.size() == 0 || - (attr.shape.size() == 1 && attr.shape[0] == 1))) - { - throw std::runtime_error( - "[ADIOS2] Expecting scalar ADIOS variable, got " + - std::to_string(attr.shape.size()) + "D: " + name); - } - *resource = *attr.data; - return determineDatatype(); - } - - template - void AttributeTypes>::createAttribute( - adios2::IO &IO, - adios2::Engine &engine, - detail::BufferedAttributeWrite ¶ms, - const std::vector &value) - { - auto size = value.size(); - auto attr = IO.InquireVariable(params.name); - // @todo check size - if (!attr) - { - attr = IO.DefineVariable(params.name, {size}, {0}, {size}); - } - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed defining variable '" + - params.name + "'."); - } - engine.Put(attr, value.data(), adios2::Mode::Deferred); - } - - template - Datatype AttributeTypes>::readAttribute( - detail::PreloadAdiosAttributes const &preloadedAttributes, - std::string name, - std::shared_ptr resource) - { - detail::AttributeWithShape attr = - preloadedAttributes.getAttribute(name); - if (attr.shape.size() != 1) - { - throw std::runtime_error("[ADIOS2] Expecting 1D ADIOS variable"); - } - - std::vector res(attr.shape[0]); - std::copy_n(attr.data, attr.shape[0], res.data()); - *resource = std::move(res); - return determineDatatype>(); - } - - void AttributeTypes>::createAttribute( - adios2::IO &IO, - adios2::Engine &engine, - detail::BufferedAttributeWrite ¶ms, - const std::vector &vec) - { - size_t width = 0; - for (auto const &str : vec) - { - width = std::max(width, str.size()); - } - ++width; // null delimiter - size_t const height = vec.size(); - - auto attr = IO.InquireVariable(params.name); - // @todo check size - if (!attr) - { - attr = IO.DefineVariable( - params.name, {height, width}, {0, 0}, {height, width}); - } - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed defining variable '" + - params.name + "'."); - } - - // write this thing to the params, so we don't get a use after free - // due to deferred writing - params.bufferForVecString = std::vector(width * height, 0); - for (size_t i = 0; i < height; ++i) - { - size_t start = i * width; - std::string const &str = vec[i]; - std::copy( - str.begin(), - str.end(), - params.bufferForVecString.begin() + start); - } - - engine.Put( - attr, params.bufferForVecString.data(), adios2::Mode::Deferred); - } - - Datatype AttributeTypes>::readAttribute( - detail::PreloadAdiosAttributes const &preloadedAttributes, - std::string name, - std::shared_ptr resource) - { - /* - * char_type parameter only for specifying the "template" type. - */ - auto loadFromDatatype = [&preloadedAttributes, &name, &resource]( - auto char_type) { - using char_t = decltype(char_type); - detail::AttributeWithShape attr = - preloadedAttributes.getAttribute(name); - if (attr.shape.size() != 2) - { - throw std::runtime_error( - "[ADIOS2] Expecting 2D ADIOS variable"); - } - char_t const *loadedData = attr.data; - size_t height = attr.shape[0]; - size_t width = attr.shape[1]; - - std::vector res(height); - if (std::is_signed::value == std::is_signed::value) - { - /* - * This branch is chosen if the signedness of the - * ADIOS variable corresponds with the signedness of the - * char type on the current platform. - * In this case, the C++ standard guarantees that the - * representations for char and (un)signed char are - * identical, reinterpret_cast-ing the loadedData to - * char in order to construct our strings will be fine. - */ - for (size_t i = 0; i < height; ++i) - { - size_t start = i * width; - char const *start_ptr = - reinterpret_cast(loadedData + start); - size_t j = 0; - while (j < width && start_ptr[j] != 0) - { - ++j; - } - std::string &str = res[i]; - str.append(start_ptr, start_ptr + j); - } - } - else - { - /* - * This branch is chosen if the signedness of the - * ADIOS variable is different from the signedness of the - * char type on the current platform. - * In this case, we play it safe, and explicitly convert - * the loadedData to char pointwise. - */ - std::vector converted(width); - for (size_t i = 0; i < height; ++i) - { - size_t start = i * width; - auto const *start_ptr = loadedData + start; - size_t j = 0; - while (j < width && start_ptr[j] != 0) - { - converted[j] = start_ptr[j]; - ++j; - } - std::string &str = res[i]; - str.append(converted.data(), converted.data() + j); - } - } - - *resource = res; - }; - /* - * If writing char variables in ADIOS2, they might become either int8_t, - * uint8_t or char on disk depending on platform, ADIOS2 version and - * ADIOS2 engine. - * So allow reading from all these types. - */ - switch (preloadedAttributes.attributeType(name)) - { - /* - * Workaround for two bugs at once: - * ADIOS2 does not have an explicit char type, - * we don't have an explicit schar type. - * Until this is fixed, we use CHAR to represent ADIOS signed char. - * @todo revisit this workaround and its necessity - */ - case Datatype::CHAR: { - loadFromDatatype(char{}); - break; - } - case Datatype::UCHAR: { - using uchar_t = unsigned char; - loadFromDatatype(uchar_t{}); - break; - } - case Datatype::SCHAR: { - using schar_t = signed char; - loadFromDatatype(schar_t{}); - break; - } - default: { - throw std::runtime_error( - "[ADIOS2] Expecting 2D ADIOS variable of any char type."); - } - } - return Datatype::VEC_STRING; - } - - template - void AttributeTypes>::createAttribute( - adios2::IO &IO, - adios2::Engine &engine, - detail::BufferedAttributeWrite ¶ms, - const std::array &value) - { - auto attr = IO.InquireVariable(params.name); - // @todo check size - if (!attr) - { - attr = IO.DefineVariable(params.name, {n}, {0}, {n}); - } - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed defining variable '" + - params.name + "'."); - } - engine.Put(attr, value.data(), adios2::Mode::Deferred); - } - - template - Datatype AttributeTypes>::readAttribute( - detail::PreloadAdiosAttributes const &preloadedAttributes, - std::string name, - std::shared_ptr resource) - { - detail::AttributeWithShape attr = - preloadedAttributes.getAttribute(name); - if (attr.shape.size() != 1 || attr.shape[0] != n) - { - throw std::runtime_error( - "[ADIOS2] Expecting 1D ADIOS variable of extent " + - std::to_string(n)); - } - - std::array res; - std::copy_n(attr.data, n, res.data()); - *resource = std::move(res); - return determineDatatype>(); - } - - void AttributeTypes::createAttribute( - adios2::IO &IO, - adios2::Engine &engine, - detail::BufferedAttributeWrite ¶ms, - const bool value) - { - IO.DefineAttribute( - ADIOS2Defaults::str_isBooleanNewLayout + params.name, 1); - AttributeTypes::createAttribute( - IO, engine, params, toRep(value)); - } - - Datatype AttributeTypes::readAttribute( - detail::PreloadAdiosAttributes const &preloadedAttributes, - std::string name, - std::shared_ptr resource) - { - detail::AttributeWithShape attr = - preloadedAttributes.getAttribute(name); - if (!(attr.shape.size() == 0 || - (attr.shape.size() == 1 && attr.shape[0] == 1))) - { - throw std::runtime_error( - "[ADIOS2] Expecting scalar ADIOS variable, got " + - std::to_string(attr.shape.size()) + "D: " + name); - } - - *resource = fromRep(*attr.data); - return Datatype::BOOL; - } - void BufferedGet::run(BufferedActions &ba) { switchAdios2VariableType( @@ -2449,51 +2084,6 @@ namespace detail switchAdios2VariableType(dtype, *this, ba); } - void OldBufferedAttributeRead::run(BufferedActions &ba) - { - auto type = attributeInfo(ba.m_IO, name, /* verbose = */ true); - - if (type == Datatype::UNDEFINED) - { - throw error::ReadError( - error::AffectedObject::Attribute, - error::Reason::NotFound, - "ADIOS2", - name); - } - - Datatype ret = switchType( - type, ba.m_IO, name, param.resource); - *param.dtype = ret; - } - - void BufferedAttributeRead::run(BufferedActions &ba) - { - auto type = attributeInfo( - ba.m_IO, - name, - /* verbose = */ true, - VariableOrAttribute::Variable); - - if (type == Datatype::UNDEFINED) - { - throw error::ReadError( - error::AffectedObject::Attribute, - error::Reason::NotFound, - "ADIOS2", - name); - } - - Datatype ret = switchType( - type, ba.m_IO, ba.preloadAttributes, name, param.resource); - *param.dtype = ret; - } - - void BufferedAttributeWrite::run(BufferedActions &fileData) - { - switchType(dtype, *this, fileData); - } - BufferedActions::BufferedActions( ADIOS2IOHandlerImpl &impl, InvalidatableFile file) : m_file(impl.fullPath(std::move(file))) @@ -2537,18 +2127,13 @@ namespace detail return; } // if write accessing, ensure that the engine is opened - // and that all attributes are written + // and that all datasets are written // (attributes and unique_ptr datasets are written upon closing a step // or a file which users might never do) - bool needToWrite = - !m_attributeWrites.empty() || !m_uniquePtrPuts.empty(); - if ((needToWrite || !m_engine) && m_mode != adios2::Mode::Read) + bool needToWrite = !m_uniquePtrPuts.empty(); + if ((needToWrite || !m_engine) && writeOnly(m_mode)) { getEngine(); - for (auto &pair : m_attributeWrites) - { - pair.second.run(*this); - } for (auto &entry : m_uniquePtrPuts) { entry.run(*this); @@ -2658,8 +2243,8 @@ namespace detail return false; } - bool - useStepsInWriting(SupportedSchema schema, std::string const &engineType) + bool useStepsInWriting( + UseGroupTable groupTable, std::string const &engineType) { if (engineType == "bp5") { @@ -2674,11 +2259,11 @@ namespace detail case PerstepParsing::Required: return true; case PerstepParsing::Supported: - switch (schema) + switch (groupTable) { - case SupportedSchema::s_0000_00_00: + case UseGroupTable::No: return false; - case SupportedSchema::s_2021_02_09: + case UseGroupTable::Yes: return true; } break; @@ -2689,6 +2274,18 @@ namespace detail } } // namespace + size_t BufferedActions::currentStep() + { + if (nonpersistentEngine(m_engineType)) + { + return m_currentStep; + } + else + { + return getEngine().CurrentStep(); + } + } + void BufferedActions::configure_IO_Read( std::optional userSpecifiedUsesteps) { @@ -2733,10 +2330,10 @@ namespace detail * Note that in BP4 with linear access mode, we set the * StreamReader option, disabling upfrontParsing capability. * So, this branch is only taken by niche engines, such as - * BP3 or HDF5, or by BP5 with old ADIOS2 schema and normal read + * BP3 or HDF5, or by BP5 without group table and normal read * mode. Need to fall back to random access parsing. */ -#if HAS_ADIOS_2_8 +#if openPMD_HAS_ADIOS_2_8 m_mode = adios2::Mode::ReadRandomAccess; #endif break; @@ -2784,15 +2381,12 @@ namespace detail std::optional userSpecifiedUsesteps) { optimizeAttributesStreaming = - // Optimizing attributes in streaming mode is not needed in - // the variable-based ADIOS2 schema - schema() == SupportedSchema::s_0000_00_00 && // Also, it should only be done when truly streaming, not // when using a disk-based engine that behaves like a // streaming engine (otherwise attributes might vanish) nonpersistentEngine(m_engineType); - bool useSteps = useStepsInWriting(schema(), m_engineType); + bool useSteps = useStepsInWriting(useGroupTable(), m_engineType); if (userSpecifiedUsesteps.has_value()) { useSteps = userSpecifiedUsesteps.value(); @@ -2810,21 +2404,45 @@ namespace detail void BufferedActions::configure_IO(ADIOS2IOHandlerImpl &impl) { - // step/variable-based iteration encoding requires the new schema - // but new schema is available only in ADIOS2 >= v2.8 - // use old schema to support at least one single iteration otherwise - if (!m_impl->m_schema.has_value()) + // step/variable-based iteration encoding requires use of group tables + // but the group table feature is available only in ADIOS2 >= v2.9 + // use old layout to support at least one single iteration otherwise + // these properties are inferred from the opened dataset in read mode + if (writeOnly(m_mode)) { - switch (m_impl->m_iterationEncoding) + +#if openPMD_HAS_ADIOS_2_9 + if (!m_impl->m_useGroupTable.has_value()) { - case IterationEncoding::variableBased: - m_impl->m_schema = ADIOS2Schema::schema_2021_02_09; - break; - case IterationEncoding::groupBased: - case IterationEncoding::fileBased: - m_impl->m_schema = ADIOS2Schema::schema_0000_00_00; - break; + switch (m_impl->m_iterationEncoding) + { + case IterationEncoding::variableBased: + m_impl->m_useGroupTable = UseGroupTable::Yes; + break; + case IterationEncoding::groupBased: + case IterationEncoding::fileBased: + m_impl->m_useGroupTable = UseGroupTable::No; + break; + } + } + + if (m_impl->m_modifiableAttributes == + ADIOS2IOHandlerImpl::ModifiableAttributes::Unspecified) + { + m_impl->m_modifiableAttributes = m_impl->m_iterationEncoding == + IterationEncoding::variableBased + ? ADIOS2IOHandlerImpl::ModifiableAttributes::Yes + : ADIOS2IOHandlerImpl::ModifiableAttributes::No; + } +#else + if (!m_impl->m_useGroupTable.has_value()) + { + m_impl->m_useGroupTable = UseGroupTable::No; } + + m_impl->m_modifiableAttributes = + ADIOS2IOHandlerImpl::ModifiableAttributes::No; +#endif } // set engine type @@ -2892,8 +2510,7 @@ namespace detail } auto _useAdiosSteps = impl.config(ADIOS2Defaults::str_usesteps, engineConfig); - if (!_useAdiosSteps.json().is_null() && - m_mode != adios2::Mode::Read) + if (!_useAdiosSteps.json().is_null() && writeOnly(m_mode)) { userSpecifiedUsesteps = std::make_optional(_useAdiosSteps.json().get()); @@ -2942,14 +2559,7 @@ namespace detail configure_IO_Read(userSpecifiedUsesteps); break; case Access::READ_WRITE: - if ( -#if HAS_ADIOS_2_8 - m_mode == adios2::Mode::Read || - m_mode == adios2::Mode::ReadRandomAccess -#else - m_mode == adios2::Mode::Read -#endif - ) + if (readOnly(m_mode)) { configure_IO_Read(userSpecifiedUsesteps); } @@ -3088,6 +2698,23 @@ namespace detail getEngine(); } + UseGroupTable BufferedActions::detectGroupTable() + { + auto const &attributes = availableAttributes(); + auto lower_bound = + attributes.lower_bound(ADIOS2Defaults::str_activeTablePrefix); + if (lower_bound != attributes.end() && + auxiliary::starts_with( + lower_bound->first, ADIOS2Defaults::str_activeTablePrefix)) + { + return UseGroupTable::Yes; + } + else + { + return UseGroupTable::No; + } + } + adios2::Engine &BufferedActions::getEngine() { if (!m_engine) @@ -3116,17 +2743,14 @@ namespace detail adios2::Engine(m_IO.Open(m_file, tempMode))); break; } -#if HAS_ADIOS_2_8 +#if openPMD_HAS_ADIOS_2_8 case adios2::Mode::ReadRandomAccess: #endif case adios2::Mode::Read: { m_engine = std::make_optional( adios2::Engine(m_IO.Open(m_file, m_mode))); /* - * First round: decide attribute layout. - * This MUST occur before the `switch(streamStatus)` construct - * since the streamStatus might be changed after taking a look - * at the used schema. + * First round: detect use of group table */ bool openedANewStep = false; { @@ -3136,7 +2760,7 @@ namespace detail /* * In BP5 with Linear read mode, we now need to * tentatively open the first IO step. - * Otherwise we don't see the schema attribute. + * Otherwise we don't see the group table attributes. * This branch is also taken by Streaming engines. */ if (m_engine->BeginStep() != adios2::StepStatus::OK) @@ -3147,15 +2771,31 @@ namespace detail } openedANewStep = true; } - auto attr = m_IO.InquireAttribute( - ADIOS2Defaults::str_adios2Schema); - if (!attr) + + if (m_impl->m_useGroupTable.has_value()) { - m_impl->m_schema = ADIOS2Schema::schema_0000_00_00; + switch (m_impl->m_useGroupTable.value()) + { + case UseGroupTable::Yes: { + auto detectedGroupTable = detectGroupTable(); + if (detectedGroupTable == UseGroupTable::No) + { + std::cerr + << "[Warning] User requested use of group " + "table when reading from ADIOS2 " + "dataset, but no group table has been " + "found. Will ignore." + << std::endl; + m_impl->m_useGroupTable = UseGroupTable::No; + } + } + case openPMD::UseGroupTable::No: + break; + } } else { - m_impl->m_schema = attr.Data()[0]; + m_impl->m_useGroupTable = detectGroupTable(); } }; @@ -3206,11 +2846,11 @@ namespace detail } case StreamStatus::NoStream: // using random-access mode - case StreamStatus::DuringStep: - // IO step might have sneakily been opened - // by setLayoutVersion(), because otherwise we don't see - // the schema attribute break; + case StreamStatus::DuringStep: + throw error::Internal( + "[ADIOS2] Control flow error: stream status cannot be " + "DuringStep before opening the engine."); case StreamStatus::OutsideOfStep: if (openedANewStep) { @@ -3226,11 +2866,6 @@ namespace detail default: throw std::runtime_error("[ADIOS2] Control flow error!"); } - - if (attributeLayout() == AttributeLayout::ByAdiosVariables) - { - preloadAttributes.preloadAttributes(m_IO, m_engine.value()); - } break; } default: @@ -3265,11 +2900,6 @@ namespace detail // pass break; } - if (m_mode == adios2::Mode::Read && - attributeLayout() == AttributeLayout::ByAdiosVariables) - { - preloadAttributes.preloadAttributes(m_IO, m_engine.value()); - } streamStatus = StreamStatus::DuringStep; } return eng; @@ -3311,21 +2941,6 @@ namespace detail m_alreadyEnqueued.emplace_back(std::move(task)); } m_buffer.clear(); - - // m_attributeWrites and m_attributeReads are for implementing the - // 2021 ADIOS2 schema which will go anyway. - // So, this ugliness here is temporary. - for (auto &task : m_attributeWrites) - { - m_alreadyEnqueued.emplace_back(std::unique_ptr{ - new BufferedAttributeWrite{std::move(task.second)}}); - } - m_attributeWrites.clear(); - /* - * An AttributeRead is not a deferred action, so we can clear it - * immediately. - */ - m_attributeReads.clear(); throw; } } @@ -3353,10 +2968,7 @@ namespace detail */ if (streamStatus == StreamStatus::OutsideOfStep) { - if (m_buffer.empty() && - (!writeLatePuts || - (m_attributeWrites.empty() && m_uniquePtrPuts.empty())) && - m_attributeReads.empty()) + if (m_buffer.empty() && (!writeLatePuts || m_uniquePtrPuts.empty())) { if (flushUnconditionally) { @@ -3376,29 +2988,20 @@ namespace detail if (!initializedDefaults) { - m_IO.DefineAttribute( - ADIOS2Defaults::str_adios2Schema, m_impl->m_schema.value()); + // Currently only schema 0 supported + m_IO.DefineAttribute(ADIOS2Defaults::str_adios2Schema, 0); initializedDefaults = true; } if (writeLatePuts) { - for (auto &pair : m_attributeWrites) - { - pair.second.run(*this); - } for (auto &entry : m_uniquePtrPuts) { entry.run(*this); } } -#if HAS_ADIOS_2_8 - if (this->m_mode == adios2::Mode::Read || - this->m_mode == adios2::Mode::ReadRandomAccess) -#else - if (this->m_mode == adios2::Mode::Read) -#endif + if (readOnly(m_mode)) { level = FlushLevel::UserFlush; } @@ -3412,15 +3015,9 @@ namespace detail m_alreadyEnqueued.clear(); if (writeLatePuts) { - m_attributeWrites.clear(); m_uniquePtrPuts.clear(); } - for (BufferedAttributeRead &task : m_attributeReads) - { - task.run(*this); - } - m_attributeReads.clear(); break; case FlushLevel::InternalFlush: @@ -3501,21 +3098,13 @@ namespace detail flushParams, [decideFlushAPICall = std::move(decideFlushAPICall)]( BufferedActions &ba, adios2::Engine &eng) { - switch (ba.m_mode) + if (writeOnly(ba.m_mode)) { - case adios2::Mode::Write: - case adios2::Mode::Append: decideFlushAPICall(eng); - break; - case adios2::Mode::Read: -#if HAS_ADIOS_2_8 - case adios2::Mode::ReadRandomAccess: -#endif + } + else + { eng.PerformGets(); - break; - default: - throw error::Internal("[ADIOS2] Unexpected access mode."); - break; } }, writeLatePuts, @@ -3533,8 +3122,7 @@ namespace detail // sic! no else if (streamStatus == StreamStatus::NoStream) { - if ((m_mode == adios2::Mode::Write || - m_mode == adios2::Mode::Append) && + if (writeOnly(m_mode) && !m_IO.InquireAttribute( ADIOS2Defaults::str_usesstepsAttribute)) { @@ -3555,8 +3143,7 @@ namespace detail * The usessteps tag should only be set when the Series is *logically* * using steps. */ - if (calledExplicitly && - (m_mode == adios2::Mode::Write || m_mode == adios2::Mode::Append) && + if (calledExplicitly && writeOnly(m_mode) && !m_IO.InquireAttribute( ADIOS2Defaults::str_usesstepsAttribute)) { @@ -3593,6 +3180,7 @@ namespace detail uncommittedAttributes.clear(); m_updateSpans.clear(); streamStatus = StreamStatus::OutsideOfStep; + ++m_currentStep; return AdvanceStatus::OK; } case AdvanceMode::BEGINSTEP: { @@ -3601,12 +3189,6 @@ namespace detail if (streamStatus != StreamStatus::DuringStep) { adiosStatus = getEngine().BeginStep(); - if (adiosStatus == adios2::StepStatus::OK && - m_mode == adios2::Mode::Read && - attributeLayout() == AttributeLayout::ByAdiosVariables) - { - preloadAttributes.preloadAttributes(m_IO, m_engine.value()); - } } else { @@ -3629,6 +3211,7 @@ namespace detail } invalidateAttributesMap(); invalidateVariablesMap(); + m_pathsMarkedAsActive.clear(); return res; } } @@ -3720,6 +3303,48 @@ namespace detail } } + void BufferedActions::markActive(Writable *writable) + { + switch (useGroupTable()) + { + case UseGroupTable::No: + break; + case UseGroupTable::Yes: +#if openPMD_HAS_ADIOS_2_9 + { + if (writeOnly(m_mode)) + { + requireActiveStep(); + auto currentStepBuffered = currentStep(); + do + { + using attr_t = unsigned long long; + auto filePos = m_impl->setAndGetFilePosition( + writable, /* write = */ false); + auto fullPath = ADIOS2Defaults::str_activeTablePrefix + + filePos->location; + m_IO.DefineAttribute( + fullPath, + currentStepBuffered, + /* variableName = */ "", + /* separator = */ "/", + /* allowModification = */ true); + m_pathsMarkedAsActive.emplace(writable); + writable = writable->parent; + } while (writable && + m_pathsMarkedAsActive.find(writable) == + m_pathsMarkedAsActive.end()); + } + } +#else + (void)writable; + throw error::OperationUnsupportedInBackend( + m_impl->m_handler->backendName(), + "Group table feature requires ADIOS2 >= v2.9."); +#endif + break; + } + } } // namespace detail #if openPMD_HAVE_MPI diff --git a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp deleted file mode 100644 index 6df3ca8c95..0000000000 --- a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp +++ /dev/null @@ -1,282 +0,0 @@ -/* Copyright 2020-2021 Franz Poeschel - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ - -#include "openPMD/config.hpp" -#if openPMD_HAVE_ADIOS2 - -#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" - -#include "openPMD/Datatype.hpp" -#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" -#include "openPMD/auxiliary/StringManip.hpp" - -#include -#include -#include -#include -#include - -namespace openPMD::detail -{ -namespace -{ - struct GetAlignment - { - template - static constexpr size_t call() - { - return alignof(T); - } - - template - static constexpr size_t call(Args &&...) - { - return alignof(std::max_align_t); - } - }; - - struct GetSize - { - template - static constexpr size_t call() - { - return sizeof(T); - } - - template - static constexpr size_t call(Args &&...) - { - return 0; - } - }; - - struct ScheduleLoad - { - template - static void call( - adios2::IO &IO, - adios2::Engine &engine, - std::string const &name, - char *buffer, - PreloadAdiosAttributes::AttributeLocation &location) - { - adios2::Variable var = IO.InquireVariable(name); - if (!var) - { - throw std::runtime_error( - "[ADIOS2] Variable not found: " + name); - } - adios2::Dims const &shape = location.shape; - adios2::Dims offset(shape.size(), 0); - if (shape.size() > 0) - { - var.SetSelection({offset, shape}); - } - T *dest = reinterpret_cast(buffer); - size_t numItems = 1; - for (auto extent : shape) - { - numItems *= extent; - } - /* - * MSVC does not like placement new of arrays, so we do it - * in a loop instead. - * https://developercommunity.visualstudio.com/t/c-placement-new-is-incorrectly-compiled/206439 - */ - for (size_t i = 0; i < numItems; ++i) - { - new (dest + i) T(); - } - location.destroy = buffer; - engine.Get(var, dest, adios2::Mode::Deferred); - } - - static constexpr char const *errorMsg = "ADIOS2"; - }; - - struct VariableShape - { - template - static adios2::Dims call(adios2::IO &IO, std::string const &name) - { - auto var = IO.InquireVariable(name); - if (!var) - { - throw std::runtime_error( - "[ADIOS2] Variable not found: " + name); - } - return var.Shape(); - } - - template - static adios2::Dims call(Args &&...) - { - return {}; - } - }; - - struct AttributeLocationDestroy - { - template - static void call(char *ptr, size_t numItems) - { - T *destroy = reinterpret_cast(ptr); - for (size_t i = 0; i < numItems; ++i) - { - destroy[i].~T(); - } - } - - template - static void call(Args &&...) - {} - }; -} // namespace - -using AttributeLocation = PreloadAdiosAttributes::AttributeLocation; - -AttributeLocation::AttributeLocation( - adios2::Dims shape_in, size_t offset_in, Datatype dt_in) - : shape(std::move(shape_in)), offset(offset_in), dt(dt_in) -{} - -AttributeLocation::AttributeLocation(AttributeLocation &&other) - : shape{std::move(other.shape)} - , offset{std::move(other.offset)} - , dt{std::move(other.dt)} - , destroy{std::move(other.destroy)} -{ - other.destroy = nullptr; -} - -AttributeLocation &AttributeLocation::operator=(AttributeLocation &&other) -{ - this->shape = std::move(other.shape); - this->offset = std::move(other.offset); - this->dt = std::move(other.dt); - this->destroy = std::move(other.destroy); - other.destroy = nullptr; - return *this; -} - -PreloadAdiosAttributes::AttributeLocation::~AttributeLocation() -{ - /* - * If the object has been moved from, this may be empty. - * Or else, if no custom destructor has been emplaced. - */ - if (destroy) - { - size_t length = 1; - for (auto ext : shape) - { - length *= ext; - } - switchAdios2AttributeType( - dt, destroy, length); - } -} - -void PreloadAdiosAttributes::preloadAttributes( - adios2::IO &IO, adios2::Engine &engine) -{ - m_offsets.clear(); - std::map > attributesByType; - auto addAttribute = [&attributesByType](Datatype dt, std::string name) { - constexpr size_t reserve = 10; - auto it = attributesByType.find(dt); - if (it == attributesByType.end()) - { - it = attributesByType.emplace_hint( - it, dt, std::vector()); - it->second.reserve(reserve); - } - it->second.push_back(std::move(name)); - }; - // PHASE 1: collect names of available attributes by ADIOS datatype - for (auto &variable : IO.AvailableVariables()) - { - if (auxiliary::ends_with(variable.first, "/__data__")) - { - continue; - } - // this will give us basic types only, no fancy vectors or similar - Datatype dt = fromADIOS2Type(IO.VariableType(variable.first)); - addAttribute(dt, std::move(variable.first)); - } - - // PHASE 2: get offsets for attributes in buffer - std::map offsets; - size_t currentOffset = 0; - for (auto &pair : attributesByType) - { - size_t alignment = switchAdios2AttributeType(pair.first); - size_t size = switchAdios2AttributeType(pair.first); - // go to next offset with valid alignment - size_t modulus = currentOffset % alignment; - if (modulus > 0) - { - currentOffset += alignment - modulus; - } - for (std::string &name : pair.second) - { - adios2::Dims shape = - switchAdios2AttributeType(pair.first, IO, name); - size_t elements = 1; - for (auto extent : shape) - { - elements *= extent; - } - m_offsets.emplace( - std::piecewise_construct, - std::forward_as_tuple(std::move(name)), - std::forward_as_tuple( - std::move(shape), currentOffset, pair.first)); - currentOffset += elements * size; - } - } - // now, currentOffset is the number of bytes that we need to allocate - // PHASE 3: allocate new buffer and schedule loads - m_rawBuffer.resize(currentOffset); - for (auto &pair : m_offsets) - { - switchAdios2AttributeType( - pair.second.dt, - IO, - engine, - pair.first, - &m_rawBuffer[pair.second.offset], - pair.second); - } -} - -Datatype PreloadAdiosAttributes::attributeType(std::string const &name) const -{ - auto it = m_offsets.find(name); - if (it == m_offsets.end()) - { - return Datatype::UNDEFINED; - } - return it->second.dt; -} -} // namespace openPMD::detail - -#endif // openPMD_HAVE_ADIOS2 diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 533691a060..63b998ad1d 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -1412,7 +1412,8 @@ void HDF5IOHandlerImpl::writeDataset( void HDF5IOHandlerImpl::writeAttribute( Writable *writable, Parameter const ¶meters) { - if (parameters.changesOverSteps) + if (parameters.changesOverSteps == + Parameter::ChangesOverSteps::Yes) { // cannot do this return; diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index e17e4929eb..bec6cebb71 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -786,7 +786,8 @@ void JSONIOHandlerImpl::writeDataset( void JSONIOHandlerImpl::writeAttribute( Writable *writable, Parameter const ¶meter) { - if (parameter.changesOverSteps) + if (parameter.changesOverSteps == + Parameter::ChangesOverSteps::Yes) { // cannot do this return; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 3e6560e4c9..95bfeef987 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -269,13 +269,18 @@ void Iteration::flushVariableBased( pOpen.path = ""; IOHandler()->enqueue(IOTask(this, pOpen)); /* - * In v-based encoding, the snapshot attribute must always be written, - * so don't set the `changesOverSteps` flag of the IOTask here. + * In v-based encoding, the snapshot attribute must always be written. * Reason: Even in backends that don't support changing attributes, * variable-based iteration encoding can be used to write one single * iteration. Then, this attribute determines which iteration it is. */ - this->setAttribute("snapshot", i); + Parameter wAttr; + wAttr.changesOverSteps = + Parameter::ChangesOverSteps::IfPossible; + wAttr.name = "snapshot"; + wAttr.resource = (unsigned long long)i; + wAttr.dtype = Datatype::ULONGLONG; + IOHandler()->enqueue(IOTask(this, wAttr)); } switch (flushParams.flushLevel) @@ -528,11 +533,15 @@ void Iteration::read_impl(std::string const &groupPath) #ifdef openPMD_USE_INVASIVE_TESTS if (containsAttribute("__openPMD_internal_fail")) { - throw error::ReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - {}, - "Deliberately failing this iteration for testing purposes"); + if (getAttribute("__openPMD_internal_fail").get() == + "asking for trouble") + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + {}, + "Deliberately failing this iteration for testing purposes"); + } } #endif } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index a54373be59..2d72f4c150 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -262,6 +262,8 @@ void RecordComponent::flush( { if (constant()) { + bool isVBased = retrieveSeries().iterationEncoding() == + IterationEncoding::variableBased; Parameter pCreate; pCreate.path = name; IOHandler()->enqueue(IOTask(this, pCreate)); @@ -269,11 +271,21 @@ void RecordComponent::flush( aWrite.name = "value"; aWrite.dtype = rc.m_constantValue.dtype; aWrite.resource = rc.m_constantValue.getResource(); + if (isVBased) + { + aWrite.changesOverSteps = Parameter< + Operation::WRITE_ATT>::ChangesOverSteps::IfPossible; + } IOHandler()->enqueue(IOTask(this, aWrite)); aWrite.name = "shape"; Attribute a(getExtent()); aWrite.dtype = a.dtype; aWrite.resource = a.getResource(); + if (isVBased) + { + aWrite.changesOverSteps = Parameter< + Operation::WRITE_ATT>::ChangesOverSteps::IfPossible; + } IOHandler()->enqueue(IOTask(this, aWrite)); } else @@ -291,11 +303,18 @@ void RecordComponent::flush( { if (constant()) { + bool isVBased = retrieveSeries().iterationEncoding() == + IterationEncoding::variableBased; Parameter aWrite; aWrite.name = "shape"; Attribute a(getExtent()); aWrite.dtype = a.dtype; aWrite.resource = a.getResource(); + if (isVBased) + { + aWrite.changesOverSteps = Parameter< + Operation::WRITE_ATT>::ChangesOverSteps::IfPossible; + } IOHandler()->enqueue(IOTask(this, aWrite)); } else diff --git a/src/Series.cpp b/src/Series.cpp index 99e654b33a..d93b3c3a98 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1961,7 +1961,8 @@ void Series::flushStep(bool doFlush) * one IO step. */ Parameter wAttr; - wAttr.changesOverSteps = true; + wAttr.changesOverSteps = + Parameter::ChangesOverSteps::Yes; wAttr.name = "snapshot"; wAttr.resource = std::vector{ series.m_currentlyActiveIterations.begin(), diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index b6621ea8f1..e513e5798e 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -12,6 +12,7 @@ #if openPMD_HAVE_ADIOS2 #include #define HAS_ADIOS_2_8 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208) +#define HAS_ADIOS_2_9 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209) #endif #include @@ -1137,7 +1138,9 @@ void adios2_streaming(bool variableBasedLayout) TEST_CASE("adios2_streaming", "[pseudoserial][adios2]") { +#if HAS_ADIOS_2_9 adios2_streaming(true); +#endif // HAS_ADIOS_2_9 adios2_streaming(false); } @@ -1377,10 +1380,10 @@ enum class ParseMode * Iterations are returned in ascending order. * If an IO step returns an iteration whose index is lower than the * last one, it will be skipped. - * This mode of parsing is not available for the BP4 engine with ADIOS2 - * schema 0, since BP4 does not associate attributes with the step in - * which they were created, making it impossible to separate parsing into - * single steps. + * This mode of parsing is not available for the BP4 engine without the + * group table feature, since BP4 does not associate attributes with the + * step in which they were created, making it impossible to separate parsing + * into single steps. */ LinearWithoutSnapshot, /* @@ -1543,8 +1546,8 @@ void append_mode( uint64_t iterationOrder[] = {0, 1, 2, 3, 4, 7, 10, 11}; /* * This one is a bit tricky: - * The BP4 engine has no way of parsing a Series in the old - * ADIOS2 schema step-by-step, since attributes are not + * The BP4 engine has no way of parsing a Series step-by-step in + * ADIOS2 without group tables, since attributes are not * associated with the step in which they were created. * As a result, when readIterations() is called, the whole thing * is parsed immediately ahead-of-time. @@ -1552,7 +1555,7 @@ void append_mode( * but since the IO steps don't correspond with the order of * iterations returned (there is no way to figure out that order), * we cannot load data in here. - * BP4 in the old ADIOS2 schema only supports either of the + * BP4 in ADIOS2 without group table only supports either of the * following: 1) A Series in which the iterations are present in * ascending order. 2) Or accessing the Series in READ_ONLY mode. */ @@ -1679,7 +1682,7 @@ TEST_CASE("append_mode", "[serial]") { "adios2": { - "schema": 0, + "use_group_table": false, "engine": { "usesteps" : true @@ -1690,7 +1693,7 @@ TEST_CASE("append_mode", "[serial]") { "adios2": { - "schema": 20210209, + "use_group_table": true, "engine": { "usesteps" : true @@ -1709,6 +1712,8 @@ TEST_CASE("append_mode", "[serial]") #if HAS_ADIOS_2_8 append_mode( t, false, ParseMode::LinearWithoutSnapshot, jsonConfigOld); +#endif +#if HAS_ADIOS_2_9 append_mode(t, false, ParseMode::WithSnapshot, jsonConfigNew); // This test config does not make sense // append_mode(t, true, ParseMode::WithSnapshot, jsonConfigOld); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 758db19c09..af53c11fe4 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -6,14 +6,12 @@ #define OPENPMD_protected public: #endif +#include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/openPMD.hpp" -#if openPMD_HAVE_ADIOS2 -#include -#endif #include #include @@ -88,128 +86,6 @@ std::vector testedFileExtensions() return {allExtensions.begin(), newEnd}; } -#if openPMD_HAVE_ADIOS2 -TEST_CASE("adios2_char_portability", "[serial][adios2]") -{ - /* - * This tests portability of char attributes in ADIOS2 in schema 20210209. - */ - - if (auxiliary::getEnvString("OPENPMD_NEW_ATTRIBUTE_LAYOUT", "NOT_SET") == - "NOT_SET") - { - /* - * @todo As soon as we have added automatic detection for the new - * layout, this environment variable should be ignore read-side. - * Then we can delete this if condition again. - */ - return; - } - // @todo remove new_attribute_layout key as soon as schema-based versioning - // is merged - std::string const config = R"END( -{ - "adios2": - { - "new_attribute_layout": true, - "schema": 20210209 - } -})END"; - { - adios2::ADIOS adios; - auto IO = adios.DeclareIO("IO"); - auto engine = IO.Open( - "../samples/adios2_char_portability.bp", adios2::Mode::Write); - engine.BeginStep(); - - // write default openPMD attributes - auto writeAttribute = [&engine, - &IO](std::string const &name, auto value) { - using variable_type = decltype(value); - engine.Put(IO.DefineVariable(name), value); - }; - writeAttribute("/basePath", std::string("/data/%T/")); - writeAttribute("/date", std::string("2021-02-22 11:14:00 +0000")); - writeAttribute("/iterationEncoding", std::string("groupBased")); - writeAttribute("/iterationFormat", std::string("/data/%T/")); - writeAttribute("/openPMD", std::string("1.1.0")); - writeAttribute("/openPMDextension", uint32_t(0)); - writeAttribute("/software", std::string("openPMD-api")); - writeAttribute("/softwareVersion", std::string("0.14.0-dev")); - - IO.DefineAttribute( - "__openPMD_internal/openPMD2_adios2_schema", 20210209); - IO.DefineAttribute("__openPMD_internal/useSteps", 1); - - // write char things that should be read back properly - - std::string baseString = "abcdefghi"; - // null termination not necessary, ADIOS knows the size of its variables - std::vector signedVector(9); - std::vector unsignedVector(9); - for (unsigned i = 0; i < 9; ++i) - { - signedVector[i] = baseString[i]; - unsignedVector[i] = baseString[i]; - } - engine.Put( - IO.DefineVariable( - "/signedVector", {3, 3}, {0, 0}, {3, 3}), - signedVector.data()); - engine.Put( - IO.DefineVariable( - "/unsignedVector", {3, 3}, {0, 0}, {3, 3}), - unsignedVector.data()); - engine.Put( - IO.DefineVariable( - "/unspecifiedVector", {3, 3}, {0, 0}, {3, 3}), - baseString.c_str()); - - writeAttribute("/signedChar", (signed char)'a'); - writeAttribute("/unsignedChar", (unsigned char)'a'); - writeAttribute("/char", (char)'a'); - - engine.EndStep(); - engine.Close(); - } - - { - Series read( - "../samples/adios2_char_portability.bp", Access::READ_ONLY, config); - auto signedVectorAttribute = read.getAttribute("signedVector"); - REQUIRE(signedVectorAttribute.dtype == Datatype::VEC_STRING); - auto unsignedVectorAttribute = read.getAttribute("unsignedVector"); - REQUIRE(unsignedVectorAttribute.dtype == Datatype::VEC_STRING); - auto unspecifiedVectorAttribute = - read.getAttribute("unspecifiedVector"); - REQUIRE(unspecifiedVectorAttribute.dtype == Datatype::VEC_STRING); - std::vector desiredVector{"abc", "def", "ghi"}; - REQUIRE( - signedVectorAttribute.get >() == - desiredVector); - REQUIRE( - unsignedVectorAttribute.get >() == - desiredVector); - REQUIRE( - unspecifiedVectorAttribute.get >() == - desiredVector); - - auto signedCharAttribute = read.getAttribute("signedChar"); - // we don't have that datatype yet - // REQUIRE(unsignedCharAttribute.dtype == Datatype::SCHAR); - auto unsignedCharAttribute = read.getAttribute("unsignedChar"); - REQUIRE(unsignedCharAttribute.dtype == Datatype::UCHAR); - auto charAttribute = read.getAttribute("char"); - // might currently report Datatype::UCHAR on some platforms - // REQUIRE(unsignedCharAttribute.dtype == Datatype::CHAR); - - REQUIRE(signedCharAttribute.get() == char('a')); - REQUIRE(unsignedCharAttribute.get() == char('a')); - REQUIRE(charAttribute.get() == char('a')); - } -} -#endif - namespace detail { template @@ -683,7 +559,9 @@ TEST_CASE("close_iteration_interleaved_test", "[serial]") // run this test for ADIOS2 & JSON only if (t == "h5") continue; +#if openPMD_HAS_ADIOS_2_9 close_iteration_interleaved_test(t, IterationEncoding::variableBased); +#endif // openPMD_HAS_ADIOS_2_9 } } @@ -5025,6 +4903,7 @@ TEST_CASE("bp4_steps", "[serial][adios2]") // dontUseSteps, // Access::READ_LINEAR); +#if openPMD_HAS_ADIOS_2_9 /* * Do this whole thing once more, but this time use the new attribute * layout. @@ -5032,7 +4911,7 @@ TEST_CASE("bp4_steps", "[serial][adios2]") useSteps = R"( { "adios2": { - "schema": 20210209, + "use_group_table": true, "engine": { "type": "bp4", "usesteps": true @@ -5043,7 +4922,7 @@ TEST_CASE("bp4_steps", "[serial][adios2]") dontUseSteps = R"( { "adios2": { - "schema": 20210209, + "use_group_table": true, "engine": { "type": "bp4", "usesteps": false @@ -5052,6 +4931,11 @@ TEST_CASE("bp4_steps", "[serial][adios2]") } )"; // sing the yes no song + bp4_steps( + "../samples/newlayout_bp4steps_yes_yes.bp", + useSteps, + useSteps, + Access::READ_LINEAR); bp4_steps("../samples/newlayout_bp4steps_yes_yes.bp", useSteps, useSteps); bp4_steps( "../samples/newlayout_bp4steps_yes_no.bp", useSteps, dontUseSteps); @@ -5070,6 +4954,7 @@ TEST_CASE("bp4_steps", "[serial][adios2]") useSteps, dontUseSteps, Access::READ_LINEAR); +#endif } #endif @@ -5465,12 +5350,64 @@ TEST_CASE("git_adios2_sample_test", "[serial][adios2]") } } +#if openPMD_HAS_ADIOS_2_9 +void adios2_group_table( + std::string const &jsonWrite, + std::string const &jsonRead, + bool canDeleteGroups) +{ + Series write( + "../samples/group_table.bp", + Access::CREATE, + json::merge(R"(iteration_encoding = "variable_based")", jsonWrite)); + // write E_x and E_y in iteration 0, only E_x in iteration 1 + write.writeIterations()[0].meshes["E"]["x"].makeEmpty(Datatype::FLOAT, 1); + write.writeIterations()[0].meshes["E"]["y"].makeEmpty(Datatype::FLOAT, 1); + write.writeIterations()[1].meshes["E"]["x"].makeEmpty(Datatype::FLOAT, 1); + write.close(); + + Series read("../samples/group_table.bp", Access::READ_LINEAR, jsonRead); + for (auto iteration : read.readIterations()) + { + switch (iteration.iterationIndex) + { + case 0: + REQUIRE(iteration.meshes["E"].contains("x")); + REQUIRE(iteration.meshes["E"].contains("y")); + REQUIRE(iteration.meshes["E"].size() == 2); + break; + case 1: + if (canDeleteGroups) + { + REQUIRE(iteration.meshes["E"].contains("x")); + REQUIRE(iteration.meshes["E"].size() == 1); + } + else + { + REQUIRE(iteration.meshes["E"].contains("x")); + REQUIRE(iteration.meshes["E"].contains("y")); + REQUIRE(iteration.meshes["E"].size() == 2); + } + break; + } + } +} + +TEST_CASE("adios2_group_table", "[serial]") +{ + std::string useGroupTable = R"(adios2.use_group_table = true)"; + std::string noGroupTable = R"(adios2.use_group_table = false)"; + adios2_group_table(useGroupTable, useGroupTable, true); + adios2_group_table(noGroupTable, useGroupTable, false); + adios2_group_table(useGroupTable, noGroupTable, false); + adios2_group_table(noGroupTable, noGroupTable, false); +} + void variableBasedSeries(std::string const &file) { - std::string selectADIOS2 = R"({"backend": "adios2"})"; constexpr Extent::value_type extent = 1000; - { - Series writeSeries(file, Access::CREATE, selectADIOS2); + auto testWrite = [&file](std::string const &jsonConfig) { + Series writeSeries(file, Access::CREATE, jsonConfig); writeSeries.setAttribute("some_global", "attribute"); writeSeries.setIterationEncoding(IterationEncoding::variableBased); REQUIRE( @@ -5491,6 +5428,8 @@ void variableBasedSeries(std::string const &file) "iteration_is_larger_than_two", "it truly is"); } + iteration.setAttribute("changing_value", i); + // this tests changing extents and dimensionalities // across iterations auto E_y = iteration.meshes["E"]["y"]; @@ -5512,20 +5451,33 @@ void variableBasedSeries(std::string const &file) // in others iteration.meshes["E"].setAttribute("attr_" + std::to_string(i), i); + auto constantMesh = + iteration.meshes["changing_constant"][RecordComponent::SCALAR]; + constantMesh.resetDataset({Datatype::INT, {i}}); + constantMesh.makeConstant(i); + + auto constantParticles = + iteration.particles["changing_constant"]["position"] + [RecordComponent::SCALAR]; + constantParticles.resetDataset({Datatype::INT, {i}}); + constantParticles.makeConstant(i); + iteration.close(); } - } - - REQUIRE(auxiliary::directory_exists(file)); + REQUIRE(auxiliary::directory_exists(file)); + }; - auto testRead = [&file, &extent, &selectADIOS2]( - std::string const &jsonConfig) { + auto testRead = [&file, &extent]( + std::string const &parseMode, + bool supportsModifiableAttributes) { /* * Need linear read mode to access more than a single iteration in * variable-based iteration encoding. */ Series readSeries( - file, Access::READ_LINEAR, json::merge(selectADIOS2, jsonConfig)); + file, + Access::READ_LINEAR, + json::merge(R"(backend = "adios2")", parseMode)); size_t last_iteration_index = 0; REQUIRE(!readSeries.containsAttribute("some_global")); @@ -5547,6 +5499,13 @@ void variableBasedSeries(std::string const &file) "iteration_is_larger_than_two")); } + // If modifiable attributes are unsupported, the attribute is + // written once in step 0 and then never changed + // A warning is printed upon trying to write + REQUIRE( + iteration.getAttribute("changing_value").get() == + (supportsModifiableAttributes ? iteration.iterationIndex : 0)); + auto E_x = iteration.meshes["E"]["x"]; REQUIRE(E_x.getDimensionality() == 1); REQUIRE(E_x.getExtent()[0] == extent); @@ -5576,7 +5535,7 @@ void variableBasedSeries(std::string const &file) REQUIRE( iteration.meshes["E"].containsAttribute( "attr_" + std::to_string(otherIteration)) == - (otherIteration == iteration.iterationIndex)); + (otherIteration <= iteration.iterationIndex)); } REQUIRE( iteration.meshes["E"][std::to_string(iteration.iterationIndex)] @@ -5588,21 +5547,79 @@ void variableBasedSeries(std::string const &file) "attr_" + std::to_string(iteration.iterationIndex)) .get() == int(iteration.iterationIndex)); + auto constantMesh = + iteration.meshes["changing_constant"][RecordComponent::SCALAR]; + REQUIRE( + constantMesh.getExtent() == + std::vector{iteration.iterationIndex}); + REQUIRE( + constantMesh.getAttribute("value").get() == + iteration.iterationIndex); + + auto constantParticles = + iteration.particles["changing_constant"]["position"] + [RecordComponent::SCALAR]; + REQUIRE( + constantParticles.getExtent() == + std::vector{iteration.iterationIndex}); + REQUIRE( + constantParticles.getAttribute("value").get() == + iteration.iterationIndex); + last_iteration_index = iteration.iterationIndex; } REQUIRE(last_iteration_index == 9); }; - testRead("{\"defer_iteration_parsing\": true}"); - testRead("{\"defer_iteration_parsing\": false}"); + std::string jsonConfig = R"( +{ + "backend": "adios2", + "adios2": { + "modifiable_attributes": true + } +})"; + testWrite(jsonConfig); + REQUIRE(auxiliary::directory_exists(file)); + testRead( + "{\"defer_iteration_parsing\": true}", + /*supportsModifiableAttributes = */ true); + testRead( + "{\"defer_iteration_parsing\": false}", + /*supportsModifiableAttributes = */ true); + + jsonConfig = R"( +{ + "backend": "adios2" +})"; + testWrite(jsonConfig); + testRead( + "{\"defer_iteration_parsing\": true}", + /*supportsModifiableAttributes = */ true); + testRead( + "{\"defer_iteration_parsing\": false}", + /*supportsModifiableAttributes = */ true); + + jsonConfig = R"( +{ + "backend": "adios2", + "adios2": { + "modifiable_attributes": false + } +})"; + testWrite(jsonConfig); + testRead( + "{\"defer_iteration_parsing\": true}", + /*supportsModifiableAttributes = */ false); + testRead( + "{\"defer_iteration_parsing\": false}", + /*supportsModifiableAttributes = */ false); } -#if openPMD_HAVE_ADIOS2 TEST_CASE("variableBasedSeries", "[serial][adios2]") { - variableBasedSeries("../samples/variableBasedSeries.bp"); + variableBasedSeries("../samples/variableBasedSeries.bp4"); + variableBasedSeries("../samples/variableBasedSeries.bp5"); } -#endif void variableBasedParticleData() { @@ -5642,7 +5659,7 @@ void variableBasedParticleData() { // open file for reading Series series = - Series("../samples/variableBasedParticles.bp", Access::READ_ONLY); + Series("../samples/variableBasedParticles.bp", Access::READ_LINEAR); for (IndexedIteration iteration : series.readIterations()) { @@ -5682,7 +5699,8 @@ TEST_CASE("variableBasedParticleData", "[serial][adios2]") { variableBasedParticleData(); } -#endif +#endif // openPMD_HAS_ADIOS_2_9 +#endif // openPMD_HAS_ADIOS2 #if openPMD_HAVE_ADIOS2 #ifdef ADIOS2_HAVE_BZIP2 @@ -6007,7 +6025,7 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") } #endif } -#if openPMD_HAVE_ADIOS2 +#if openPMD_HAVE_ADIOS2 && openPMD_HAS_ADIOS_2_9 iterate_nonstreaming_series( "../samples/iterate_nonstreaming_series_variablebased.bp", true, @@ -6022,8 +6040,7 @@ void adios2_bp5_no_steps(bool usesteps) { "adios2": { - "new_attribute_layout": true, - "schema": 20210209 + "use_group_table": true } })END"; { @@ -6387,6 +6404,7 @@ TEST_CASE("deferred_parsing", "[serial]") } } +#if openPMD_HAS_ADIOS_2_9 void chaotic_stream(std::string filename, bool variableBased) { /* @@ -6396,7 +6414,7 @@ void chaotic_stream(std::string filename, bool variableBased) std::string jsonConfig = R"( { "adios2": { - "schema": 20210209, + "use_group_table": true, "engine": { "usesteps": true } @@ -6464,6 +6482,7 @@ TEST_CASE("chaotic_stream", "[serial]") chaotic_stream("../samples/chaotic_stream_vbased." + t, true); } } +#endif // openPMD_HAS_ADIOS_2_9 #ifdef openPMD_USE_INVASIVE_TESTS void unfinished_iteration_test( @@ -6475,9 +6494,15 @@ void unfinished_iteration_test( std::string file = std::string("../samples/unfinished_iteration") + (encoding == IterationEncoding::fileBased ? "_%T." : ".") + ext; { + std::vector data{0, 1, 2, 3, 4}; Series write(file, Access::CREATE, config); auto it0 = write.writeIterations()[0]; + it0.meshes["E"]["x"].resetDataset({Datatype::INT, {5}}); + it0.meshes["E"]["x"].storeChunk(data, {0}, {5}); auto it5 = write.writeIterations()[5]; + it5.meshes["E"]["x"].resetDataset({Datatype::INT, {5}}); + it5.meshes["E"]["x"].storeChunk(data, {0}, {5}); + ; /* * With enabled invasive tests, this attribute will let the Iteration * fail parsing. @@ -6485,14 +6510,16 @@ void unfinished_iteration_test( it5.setAttribute("__openPMD_internal_fail", "asking for trouble"); auto it10 = write.writeIterations()[10]; Dataset ds(Datatype::INT, {10}); - auto E_x = it10.meshes["E"]["x"]; + it10.meshes["E"]["x"].resetDataset({Datatype::INT, {5}}); + it10.meshes["E"]["x"].storeChunk(data, {0}, {5}); + it10.setAttribute("__openPMD_internal_fail", "playing nice again"); auto e_density = it10.meshes["e_density"][RecordComponent::SCALAR]; auto electron_x = it10.particles["e"]["position"]["x"]; auto electron_mass = it10.particles["e"]["mass"][RecordComponent::SCALAR]; RecordComponent *resetThese[] = { - &E_x, &e_density, &electron_x, &electron_mass}; + &e_density, &electron_x, &electron_mass}; for (RecordComponent *rc : resetThese) { rc->resetDataset(ds); @@ -6571,18 +6598,20 @@ TEST_CASE("unfinished_iteration_test", "[serial]") #if openPMD_HAVE_ADIOS2 unfinished_iteration_test( "bp", IterationEncoding::groupBased, R"({"backend": "adios2"})"); +#if openPMD_HAS_ADIOS_2_9 unfinished_iteration_test( - "bp", + "bp5", IterationEncoding::variableBased, R"( { "backend": "adios2", "iteration_encoding": "variable_based", "adios2": { - "schema": 20210209 + "use_group_table": true } } )"); +#endif // openPMD_HAS_ADIOS_2_9 unfinished_iteration_test( "bp", IterationEncoding::fileBased, R"({"backend": "adios2"})"); #endif @@ -6723,10 +6752,10 @@ enum class ParseMode * Iterations are returned in ascending order. * If an IO step returns an iteration whose index is lower than the * last one, it will be skipped. - * This mode of parsing is not available for the BP4 engine with ADIOS2 - * schema 0, since BP4 does not associate attributes with the step in - * which they were created, making it impossible to separate parsing into - * single steps. + * This mode of parsing is not available for the BP4 engine without the + * group table feature, since BP4 does not associate attributes with the + * step in which they were created, making it impossible to separate parsing + * into single steps. */ LinearWithoutSnapshot, /* @@ -6902,8 +6931,8 @@ void append_mode( uint64_t iterationOrder[] = {0, 1, 2, 3, 4, 7, 10, 11}; /* * This one is a bit tricky: - * The BP4 engine has no way of parsing a Series in the old - * ADIOS2 schema step-by-step, since attributes are not + * The BP4 engine has no way of parsing a Series step-by-step in + * ADIOS2 without group tables, since attributes are not * associated with the step in which they were created. * As a result, when readIterations() is called, the whole thing * is parsed immediately ahead-of-time. @@ -6911,7 +6940,7 @@ void append_mode( * but since the IO steps don't correspond with the order of * iterations returned (there is no way to figure out that order), * we cannot load data in here. - * BP4 in the old ADIOS2 schema only supports either of the + * BP4 in ADIOS2 without group table only supports either of the * following: 1) A Series in which the iterations are present in * ascending order. 2) Or accessing the Series in READ_ONLY mode. */ @@ -7044,7 +7073,7 @@ TEST_CASE("append_mode", "[serial]") { "adios2": { - "schema": 0, + "use_group_table": false, "engine": { "usesteps" : true @@ -7055,7 +7084,7 @@ TEST_CASE("append_mode", "[serial]") { "adios2": { - "schema": 20210209, + "use_group_table": true, "engine": { "usesteps" : true @@ -7069,6 +7098,7 @@ TEST_CASE("append_mode", "[serial]") false, ParseMode::LinearWithoutSnapshot, jsonConfigOld); +#if openPMD_HAS_ADIOS_2_9 append_mode( "../samples/append/append_groupbased." + t, false, @@ -7085,6 +7115,7 @@ TEST_CASE("append_mode", "[serial]") true, ParseMode::WithSnapshot, jsonConfigNew); +#endif } else { @@ -7096,13 +7127,14 @@ TEST_CASE("append_mode", "[serial]") } } +#if openPMD_HAS_ADIOS_2_9 void append_mode_filebased(std::string const &extension) { std::string jsonConfig = R"END( { "adios2": { - "schema": 20210209, + "use_group_table": true, "engine": { "usesteps" : true @@ -7175,6 +7207,7 @@ TEST_CASE("append_mode_filebased", "[serial]") append_mode_filebased(t); } } +#endif // openPMD_HAS_ADIOS_2_8 void groupbased_read_write(std::string const &ext) {