From b8ce8a09cd9715b7cd72462bd030df9b58d1c913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 10 Dec 2024 16:27:13 +0100 Subject: [PATCH] Unify Random-Access API and Streaming API into Series::snapshots() (#1592) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Introduce SharedAttributableData * Add AbstractSeriesIterator * Derive SeriesIterator from AbstractSeriesIterator * Little fix * Introduce Snapshots.hpp * Make AbstractSeriesIterator non-virtual * Working commit for Series::snapshots() * No virtual operator[] * Remove random-accessing from iterator * Introduce AbstractSnapshotsContainer * basic random-access iteration * RandomAccessSnapshots.hpp -> snapshots/RandomAccessIterator.hpp * ReadIterations.hpp -> snapshots/StatefulIterator.hpp * SeriesIterator.hpp -> snapshots/IteratorTraits.hpp * Snapshots.hpp -> snapshots/Snapshots.hpp * Move AbstractSnapshotsContainer to ContainerTraits.hpp * Move Container implementations to ContainerImpls.(h|c)pp * Fix: parsePreference is not set in file-based iteratione encoding * Temporarily fix test * Const iteration * Extract stuff to .cpp * Reverse iteration * Commit missing Snapshots.cpp file * empty() * Revert wrong renaming ReadIterations/StatefulIterator * Rename SeriesIterator -> StatefulIterator * Add ::at, operator[] * beginStep(): always return relevant iteration indices * Basically working example for snapshots() in write access * Extract some methods to .cpp * Fully replace WriteIterations class with the new one * Fix nullpointer issue * Little fixes * Add some further API calls * Some postfix form transformations * Use snapshots() in read example 2 * Simplify ReadIterations implementation * Further cleanup * Change representation of iterations in current step * Initiate reading of group/variable-based encoding with nextStep() * Prepare internal representation to be aware of steps * Windows fixes * Adapt tests * Unify close status * Add basic test for opening after closing * Add new end() iterator representations * Reopening logic in Iterator, not yet in Series itself * Reopening fundamentally working in READ_LINEAR * Extend test still sth wrong in append_mode test, but see about this next week * For now, adapt the append_mode test * fixes * BUGFIX: modifiable attributes, maybe extract this to dev * Ensure that iterations are never parsed twice * Move currently_available_iterations to During_t * Revert "For now, adapt the append_mode test" This reverts commit 19b68ee762876cba9c04cf596664030238c56343. * Remember where we saw what iteration * Bit of cleanup * [wip] Groupbased writing: close and reopen * Further test and implement reopening of Iterations * Unused variable * some fixes to groupbased reopen test * Filebased reopen in ADIOS2 (no READ_WRITE support yet) * Now supports READ_WRITE too in filebased mode * Some exceptions for unimplemented stuff * Works in JSON and HDF5 now too * CI fixes * Virtual destructors * CI fixes continued * Some fixes for noexcept specifications * Further CI Fixes * CI FIXES * Fixes for ADIOS2 v2.7 * placate the intel compiler * noexcept details for MSVC * Fix ulimit test * Fix after rebase: dirtyRecursive * Fixes after rebase * remove conflict markers... * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Better defaults? * Parameterize Series::snapshots() * Use enum class for last commit * Add some missing minor function implementations * Don't use globbing * Add missing include * Better include structure, put Legacy stuff to Legacy headers * Bugfix * Documentation, cleanup * Add check_recursive_include script * Fixes after rebase * Fix bug that hindered files from being properly closed * Will this fix the Windows CI errors I dont think so * Use macro instead of function Proper return() is supported beginning with CMake 3.25 only * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Better document reopening options * Update close_iteration_test * Documentation --------- Co-authored-by: Pöschel Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CMakeLists.txt | 27 +- examples/10_streaming_read.cpp | 19 +- examples/10_streaming_write.cpp | 14 +- examples/2_read_serial.cpp | 6 +- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 15 + include/openPMD/IO/ADIOS/ADIOS2File.hpp | 5 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 10 +- include/openPMD/IO/Access.hpp | 2 + include/openPMD/IO/IOTask.hpp | 14 + include/openPMD/Iteration.hpp | 58 +- include/openPMD/ReadIterations.hpp | 170 +-- include/openPMD/Series.hpp | 138 ++- include/openPMD/Streaming.hpp | 1 + include/openPMD/WriteIterations.hpp | 108 +- include/openPMD/backend/Attributable.hpp | 82 +- include/openPMD/backend/Container.hpp | 6 +- include/openPMD/backend/Writable.hpp | 7 + include/openPMD/openPMD.hpp | 3 +- include/openPMD/snapshots/ContainerImpls.hpp | 133 +++ include/openPMD/snapshots/ContainerTraits.hpp | 113 ++ include/openPMD/snapshots/IteratorHelpers.hpp | 12 + include/openPMD/snapshots/IteratorTraits.hpp | 161 +++ .../snapshots/RandomAccessIterator.hpp | 85 ++ include/openPMD/snapshots/Snapshots.hpp | 163 +++ .../openPMD/snapshots/StatefulIterator.hpp | 437 ++++++++ src/IO/ADIOS/ADIOS2File.cpp | 8 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 132 ++- src/IO/AbstractIOHandler.cpp | 4 + src/IO/Access.cpp | 30 + src/Iteration.cpp | 131 ++- src/ReadIterations.cpp | 642 +----------- src/Series.cpp | 468 +++++++-- src/WriteIterations.cpp | 124 --- src/backend/Attributable.cpp | 13 +- src/backend/Writable.cpp | 5 +- src/binding/python/Series.cpp | 41 +- src/helper/list_series.cpp | 11 +- src/snapshots/ContainerImpls.cpp | 393 +++++++ src/snapshots/ContainerTraits.cpp | 114 +++ src/snapshots/IteratorHelpers.cpp | 15 + src/snapshots/IteratorTraits.cpp | 153 +++ src/snapshots/RandomAccessIterator.cpp | 86 ++ src/snapshots/Snapshots.cpp | 104 ++ src/snapshots/StatefulIterator.cpp | 964 ++++++++++++++++++ test/Files_SerialIO/SerialIOTests.hpp | 10 + test/Files_SerialIO/close_and_reopen_test.cpp | 328 ++++++ test/Files_SerialIO/filebased_write_test.cpp | 116 +++ test/ParallelIOTest.cpp | 12 +- test/SerialIOTest.cpp | 103 +- 49 files changed, 4438 insertions(+), 1358 deletions(-) create mode 100644 include/openPMD/snapshots/ContainerImpls.hpp create mode 100644 include/openPMD/snapshots/ContainerTraits.hpp create mode 100644 include/openPMD/snapshots/IteratorHelpers.hpp create mode 100644 include/openPMD/snapshots/IteratorTraits.hpp create mode 100644 include/openPMD/snapshots/RandomAccessIterator.hpp create mode 100644 include/openPMD/snapshots/Snapshots.hpp create mode 100644 include/openPMD/snapshots/StatefulIterator.hpp create mode 100644 src/IO/Access.cpp delete mode 100644 src/WriteIterations.cpp create mode 100644 src/snapshots/ContainerImpls.cpp create mode 100644 src/snapshots/ContainerTraits.cpp create mode 100644 src/snapshots/IteratorHelpers.cpp create mode 100644 src/snapshots/IteratorTraits.cpp create mode 100644 src/snapshots/RandomAccessIterator.cpp create mode 100644 src/snapshots/Snapshots.cpp create mode 100644 src/snapshots/StatefulIterator.cpp create mode 100644 test/Files_SerialIO/SerialIOTests.hpp create mode 100644 test/Files_SerialIO/close_and_reopen_test.cpp create mode 100644 test/Files_SerialIO/filebased_write_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b49848c90d..dd3e5a9efd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -397,13 +397,12 @@ set(CORE_SOURCE src/Mesh.cpp src/ParticlePatches.cpp src/ParticleSpecies.cpp - src/ReadIterations.cpp src/Record.cpp + src/ReadIterations.cpp src/RecordComponent.cpp src/Series.cpp src/UnitDimension.cpp src/version.cpp - src/WriteIterations.cpp src/auxiliary/Date.cpp src/auxiliary/Filesystem.cpp src/auxiliary/JSON.cpp @@ -415,11 +414,19 @@ set(CORE_SOURCE src/backend/PatchRecordComponent.cpp src/backend/Writable.cpp src/benchmark/mpi/OneDimensionalBlockSlicer.cpp - src/helper/list_series.cpp) + src/helper/list_series.cpp + src/snapshots/ContainerImpls.cpp + src/snapshots/ContainerTraits.cpp + src/snapshots/IteratorHelpers.cpp + src/snapshots/IteratorTraits.cpp + src/snapshots/RandomAccessIterator.cpp + src/snapshots/Snapshots.cpp + src/snapshots/StatefulIterator.cpp) set(IO_SOURCE src/IO/AbstractIOHandler.cpp src/IO/AbstractIOHandlerImpl.cpp src/IO/AbstractIOHandlerHelper.cpp + src/IO/Access.cpp src/IO/DummyIOHandler.cpp src/IO/IOTask.cpp src/IO/FlushParams.cpp @@ -773,8 +780,20 @@ if(openPMD_BUILD_TESTING) target_compile_definitions(CatchRunner PUBLIC openPMD_HAVE_MPI=1) endif() + macro(additional_testing_sources test_name out_list) + if(${test_name} STREQUAL "SerialIO") + list(APPEND ${out_list} + test/Files_SerialIO/close_and_reopen_test.cpp + test/Files_SerialIO/filebased_write_test.cpp + ) + endif() + endmacro() + foreach(testname ${openPMD_TEST_NAMES}) - add_executable(${testname}Tests test/${testname}Test.cpp) + set(ADDITIONAL_SOURCE_FILES "") + additional_testing_sources(${testname} ADDITIONAL_SOURCE_FILES) + add_executable(${testname}Tests test/${testname}Test.cpp ${ADDITIONAL_SOURCE_FILES}) + target_include_directories(${testname}Tests PRIVATE test/Files_${testname}/) openpmd_cxx_required(${testname}Tests) set_target_properties(${testname}Tests PROPERTIES COMPILE_PDB_NAME ${testname}Tests diff --git a/examples/10_streaming_read.cpp b/examples/10_streaming_read.cpp index 12392b4e70..eb02f3b393 100644 --- a/examples/10_streaming_read.cpp +++ b/examples/10_streaming_read.cpp @@ -18,6 +18,15 @@ int main() return 0; } + // Access the Series linearly. This means that upon opening the Series, no + // data is accessed yet. Instead, the single Iterations are processed + // collectively, one after the other, and data access only happens upon + // explicitly accessing an Iteration from `Series::snapshots()`. Note that + // the Container API of `Series::snapshots()` will work in a restricted mode + // compared to the `READ_RANDOM_ACCESS` access type, refer also to the + // documentation of the `Snapshots` class in `snapshots/Snapshots.hpp`. This + // restricted workflow enables performance optimizations in the backends, + // and more importantly is compatible with streaming I/O. Series series = Series("electrons.sst", Access::READ_LINEAR, R"( { "adios2": { @@ -29,15 +38,9 @@ int main() } })"); - // `Series::writeIterations()` and `Series::readIterations()` are - // intentionally restricted APIs that ensure a workflow which also works - // in streaming setups, e.g. an iteration cannot be opened again once - // it has been closed. - // `Series::iterations` can be directly accessed in random-access workflows. - for (IndexedIteration iteration : series.readIterations()) + for (auto &[index, iteration] : series.snapshots()) { - std::cout << "Current iteration: " << iteration.iterationIndex - << std::endl; + std::cout << "Current iteration: " << index << std::endl; Record electronPositions = iteration.particles["e"]["position"]; std::array loadedChunks; std::array extents; diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 2eb825ae4a..d64bee6d79 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -1,3 +1,5 @@ +#include "openPMD/Series.hpp" +#include "openPMD/snapshots/Snapshots.hpp" #include #include @@ -38,12 +40,12 @@ int main() std::shared_ptr local_data( new position_t[length], [](position_t const *ptr) { delete[] ptr; }); - // `Series::writeIterations()` and `Series::readIterations()` are - // intentionally restricted APIs that ensure a workflow which also works - // in streaming setups, e.g. an iteration cannot be opened again once - // it has been closed. - // `Series::iterations` can be directly accessed in random-access workflows. - WriteIterations iterations = series.writeIterations(); + // Create the Series with synchronous snapshots, i.e. one Iteration after + // the other. The alternative would be random-access where multiple + // Iterations can be accessed independently from one another. This more + // restricted mode enables performance optimizations in the backends, and + // more importantly is compatible with streaming I/O. + auto iterations = series.snapshots(SnapshotWorkflow::Synchronous); for (size_t i = 0; i < 100; ++i) { Iteration iteration = iterations[i]; diff --git a/examples/2_read_serial.cpp b/examples/2_read_serial.cpp index d1f613c20b..85bd5aed1c 100644 --- a/examples/2_read_serial.cpp +++ b/examples/2_read_serial.cpp @@ -34,13 +34,13 @@ int main() cout << "Read a Series with openPMD standard version " << series.openPMD() << '\n'; - cout << "The Series contains " << series.iterations.size() + cout << "The Series contains " << series.snapshots().size() << " iterations:"; - for (auto const &i : series.iterations) + for (auto const &i : series.snapshots()) cout << "\n\t" << i.first; cout << '\n'; - Iteration i = series.iterations[100]; + Iteration i = series.snapshots()[100]; cout << "Iteration 100 contains " << i.meshes.size() << " meshes:"; for (auto const &m : i.meshes) cout << "\n\t" << m.first; diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 9cb275d339..e741b029df 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -66,6 +66,21 @@ namespace adios_defs Yes, No }; + + /* + * Necessary to implement the `reopen` flag of + * `Parameter`. The distinction between Open and + * Reopen is necessary for Write workflows in file-based encoding. In order + * to write new data to an Iteration that was created and closed previously, + * the only applicable access mode is Append mode, ideally in conjunction + * with `SetParameter("FlattenSteps", "ON")`. + */ + enum class OpenFileAs + { + Create, + Open, + ReopenFileThatWeCreated + }; } // namespace adios_defs /* diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index 0bcdaa6131..5d519a85ad 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -229,7 +229,10 @@ class ADIOS2File using AttributeMap_t = std::map; - ADIOS2File(ADIOS2IOHandlerImpl &impl, InvalidatableFile file); + ADIOS2File( + ADIOS2IOHandlerImpl &impl, + InvalidatableFile file, + adios_defs::OpenFileAs); ~ADIOS2File(); diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index db3162a2da..fa300da472 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -212,7 +212,8 @@ class ADIOS2IOHandlerImpl * @brief The ADIOS2 access type to chose for Engines opened * within this instance. */ - adios2::Mode adios2AccessMode(std::string const &fullPath); + adios2::Mode + adios2AccessMode(std::string const &fullPath, adios_defs::OpenFileAs); FlushTarget m_flushTarget = FlushTarget::Disk; @@ -403,10 +404,13 @@ class ADIOS2IOHandlerImpl */ GroupOrDataset groupOrDataset(Writable *); - enum class IfFileNotOpen : bool + enum class IfFileNotOpen : char { OpenImplicitly, - ThrowError + CreateImplicitly, + ThrowError, + ReopenFileThatWeCreated, + ReopenFileFoundOnDisk = OpenImplicitly, }; detail::ADIOS2File & diff --git a/include/openPMD/IO/Access.hpp b/include/openPMD/IO/Access.hpp index eddd9f40e9..a5a2a84ae0 100644 --- a/include/openPMD/IO/Access.hpp +++ b/include/openPMD/IO/Access.hpp @@ -82,6 +82,8 @@ enum class Access APPEND //!< write new iterations to an existing series without reading }; // Access +std::ostream &operator<<(std::ostream &o, Access const &a); + namespace access { inline bool readOnly(Access access) diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 731372f9e1..5589db6923 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -187,7 +187,21 @@ struct OPENPMDAPI_EXPORT Parameter new Parameter(std::move(*this))); } + // Needed for reopening files in file-based Iteration encoding when using + // R/W-mode in ADIOS2. Files can only be opened for reading XOR writing, + // so R/W mode in file-based encoding can only operate at the granularity + // of files in ADIOS2. The frontend needs to tell us if we should reopen + // a file for continued reading (WasFoundOnDisk) or for continued writing + // (WasCreatedByUs). + enum class Reopen + { + WasCreatedByUs, + WasFoundOnDisk, + NoReopen + }; + std::string name = ""; + Reopen reopen = Reopen::NoReopen; using ParsePreference = internal::ParsePreference; std::shared_ptr out_parsePreference = std::make_shared(ParsePreference::UpFront); diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index a35ceccd74..8c39c0a518 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -48,10 +48,8 @@ namespace internal Open, //!< Iteration has not been closed ClosedInFrontend, /*!< Iteration has been closed, but task has not yet been propagated to the backend */ - ClosedInBackend, /*!< Iteration has been closed and task has been + Closed, /*!< Iteration has been closed and task has been propagated to the backend */ - ClosedTemporarily /*!< Iteration has been closed internally and may - be reopened later */ }; struct DeferredParseAccess @@ -71,11 +69,6 @@ namespace internal * (Group- and variable-based parsing shares the same code logic.) */ bool fileBased = false; - /** - * If fileBased == true, the file name (without file path) of the file - * containing this iteration. - */ - std::string filename; bool beginStep = false; }; @@ -92,6 +85,14 @@ namespace internal * overwritten. */ CloseStatus m_closed = CloseStatus::Open; + /* + * While parsing a file-based Series, each file is opened, read, then + * closed again. Explicitly `Iteration::open()`ing a file should only be + * necessary after having explicitly closed it (or in + * defer_iteration_parsing mode). So, the parsing procedures will set + * this flag as true when closing an Iteration. + */ + bool allow_reopening_implicitly = false; /** * Whether a step is currently active for this iteration. @@ -107,14 +108,6 @@ namespace internal * Otherwise empty. */ std::optional m_deferredParseAccess{}; - - /** - * Upon reading a file, set this field to the used file name. - * In inconsistent iteration paddings, we must remember the name of the - * file since it cannot be reconstructed from the filename pattern - * alone. - */ - std::optional m_overrideFilebasedFilename{}; }; } // namespace internal /** @brief Logical compilation of data from one snapshot (e.g. a single @@ -128,16 +121,18 @@ class Iteration : public Attributable template friend class Container; friend class Series; - friend class WriteIterations; - friend class SeriesIterator; friend class internal::AttributableData; template friend T &internal::makeOwning(T &self, Series); friend class Writable; + friend class StatefulIterator; + friend class StatefulSnapshotsContainer; public: Iteration(Iteration const &) = default; + Iteration(Iteration &&) = default; Iteration &operator=(Iteration const &) = default; + Iteration &operator=(Iteration &&) = default; using IterationIndex_t = uint64_t; @@ -220,12 +215,19 @@ class Iteration : public Attributable /** * @brief Has the iteration been closed? - * A closed iteration may not (yet) be reopened. * * @return Whether the iteration has been closed. */ bool closed() const; + /** + * @brief Has the iteration been parsed yet? + If not, it will contain no structure yet. + * + * @return Whether the iteration has been parsed. + */ + bool parsed() const; + /** * @brief Has the iteration been closed by the writer? * Background: Upon calling Iteration::close(), the openPMD API @@ -299,6 +301,7 @@ class Iteration : public Attributable */ void reread(std::string const &path); void readFileBased( + IterationIndex_t, std::string const &filePath, std::string const &groupPath, bool beginStep); @@ -314,7 +317,7 @@ class Iteration : public Attributable */ struct BeginStepStatus { - using AvailableIterations_t = std::optional >; + using AvailableIterations_t = std::vector; AdvanceStatus stepStatus{}; /* @@ -356,11 +359,8 @@ class Iteration : public Attributable * Useful in group-based iteration encoding where the Iteration will only * be known after opening the step. */ - static BeginStepStatus beginStep( - std::optional thisObject, - Series &series, - bool reread, - std::set const &ignoreIterations = {}); + static BeginStepStatus + beginStep(std::optional thisObject, Series &series, bool reread); /** * @brief End an IO step on the IO file (or file-like object) @@ -434,13 +434,17 @@ inline T Iteration::dt() const */ class IndexedIteration : public Iteration { - friend class SeriesIterator; - friend class WriteIterations; + friend class StatefulIterator; + friend class LegacyIteratorAdaptor; public: using index_t = Iteration::IterationIndex_t; index_t const iterationIndex; + inline IndexedIteration(std::pair pair) + : Iteration(std::move(pair.second)), iterationIndex(pair.first) + {} + private: template IndexedIteration(Iteration_t &&it, index_t index) diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/ReadIterations.hpp index 35f2f740ce..9a4eff9221 100644 --- a/include/openPMD/ReadIterations.hpp +++ b/include/openPMD/ReadIterations.hpp @@ -1,149 +1,44 @@ -/* Copyright 2021 Franz Poeschel - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ #pragma once +/* + * Legacy header. + */ + #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" -#include "openPMD/backend/ParsePreference.hpp" - -#include -#include -#include -#include +#include "openPMD/snapshots/Snapshots.hpp" namespace openPMD { -class SeriesIterator +/** @brief Legacy Iterator type for `Series::readIterations()` + * + * Wraps the Iterator type of `Series::snapshots()`, but has `IndexedIteration` + * as value_type instead of `std::pair`. + */ +class LegacyIteratorAdaptor { - using iteration_index_t = IndexedIteration::index_t; - - using maybe_series_t = std::optional; - - struct SharedData - { - SharedData() = default; - SharedData(SharedData const &) = delete; - SharedData(SharedData &&) = delete; - SharedData &operator=(SharedData const &) = delete; - SharedData &operator=(SharedData &&) = delete; - - maybe_series_t series; - std::deque iterationsInCurrentStep; - uint64_t currentIteration{}; - std::optional parsePreference; - /* - * Necessary because in the old ADIOS2 schema, old iterations' metadata - * will leak into new steps, making the frontend think that the groups - * are still there and the iterations can be parsed again. - */ - std::set ignoreIterations; - }; - - /* - * The shared data is never empty, emptiness is indicated by std::optional - */ - std::shared_ptr> m_data = - std::make_shared>(std::nullopt); - - SharedData &get() - { - return m_data->value(); - } - SharedData const &get() const - { - return m_data->value(); - } - -public: - //! construct the end() iterator - explicit SeriesIterator(); - - SeriesIterator( - Series const &, - std::optional const &parsePreference); - - SeriesIterator &operator++(); - - IndexedIteration operator*(); - - bool operator==(SeriesIterator const &other) const; - - bool operator!=(SeriesIterator const &other) const; - - static SeriesIterator end(); + using value_type = IndexedIteration; + using parent_t = Snapshots::iterator; private: - inline bool setCurrentIteration() - { - auto &data = get(); - if (data.iterationsInCurrentStep.empty()) - { - std::cerr << "[ReadIterations] Encountered a step without " - "iterations. Closing the Series." - << std::endl; - *this = end(); - return false; - } - data.currentIteration = *data.iterationsInCurrentStep.begin(); - return true; - } - - inline std::optional peekCurrentIteration() - { - auto &data = get(); - if (data.iterationsInCurrentStep.empty()) - { - return std::nullopt; - } - else - { - return {*data.iterationsInCurrentStep.begin()}; - } - } + friend class ReadIterations; + Snapshots::iterator m_iterator; + LegacyIteratorAdaptor(Snapshots::iterator iterator); - std::optional nextIterationInStep(); - - /* - * When a step cannot successfully be opened, the method nextStep() calls - * itself again recursively. - * (Recursion massively simplifies the logic here, and it only happens - * in case of error.) - * After successfully beginning a step, this methods needs to remember, how - * many broken steps have been skipped. In case the Series does not use - * the /data/snapshot attribute, this helps figuring out which iteration - * is now active. Hence, recursion_depth. - */ - std::optional nextStep(size_t recursion_depth); - - std::optional loopBody(); - - void deactivateDeadIteration(iteration_index_t); - - void initSeriesInLinearReadMode(); - - void close(); +public: + value_type operator*() const; + LegacyIteratorAdaptor &operator++(); + bool operator==(LegacyIteratorAdaptor const &other) const; + bool operator!=(LegacyIteratorAdaptor const &other) const; }; /** - * @brief Reading side of the streaming API. + * @brief Legacy class as return type for `Series::readIterations()`. + * + * This is a feature-restricted subset for the functionality of + * `Series::snapshots()`, prefer using that. The compatibility layer is needed + * due to the different value_type for `Series::readIterations()`-based + * iteration (`IndexedIteration` instead of `std::pair`). * * Create instance via Series::readIterations(). * For use in a C++11-style foreach loop over iterations. @@ -152,10 +47,8 @@ class SeriesIterator * Calling Iteration::close() manually before opening the next iteration is * encouraged and will implicitly flush all deferred IO actions. * Otherwise, Iteration::close() will be implicitly called upon - * SeriesIterator::operator++(), i.e. upon going to the next iteration in + * StatefulIterator::operator++(), i.e. upon going to the next iteration in * the foreach loop. - * Since this is designed for streaming mode, reopening an iteration is - * not possible once it has been closed. * */ class ReadIterations @@ -164,7 +57,7 @@ class ReadIterations private: using iterations_t = decltype(internal::SeriesData::iterations); - using iterator_t = SeriesIterator; + using iterator_t = LegacyIteratorAdaptor; Series m_series; std::optional m_parsePreference; @@ -175,8 +68,7 @@ class ReadIterations std::optional parsePreference); public: - iterator_t begin(); - - iterator_t end(); + auto begin() -> iterator_t; + static auto end() -> iterator_t; }; } // namespace openPMD diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index dbad461dc1..c387c803f7 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -27,13 +27,13 @@ #include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/Streaming.hpp" -#include "openPMD/WriteIterations.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/ParsePreference.hpp" #include "openPMD/config.hpp" +#include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/version.hpp" #if openPMD_HAVE_MPI @@ -50,7 +50,9 @@ #include #include #include +#include #include +#include // expose private and protected members for invasive testing #ifndef OPENPMD_private @@ -60,7 +62,7 @@ namespace openPMD { class ReadIterations; -class SeriesIterator; +class StatefulIterator; class Series; class Series; @@ -94,14 +96,6 @@ namespace internal using IterationsContainer_t = Container; IterationsContainer_t iterations{}; - /** - * For each instance of Series, there is only one instance - * of WriteIterations, stored in this Option. - * This ensures that Series::writeIteration() always returns - * the same instance. - */ - std::optional m_writeIterations; - /** * Series::readIterations() returns an iterator type that modifies the * state of the Series (by proceeding through IO steps). @@ -117,13 +111,33 @@ namespace internal * Due to include order, this member needs to be a pointer instead of * an optional. */ - std::unique_ptr m_sharedStatefulIterator; + std::unique_ptr m_sharedStatefulIterator; /** * For writing: Remember which iterations have been written in the * currently active output step. Use this later when writing the * snapshot attribute. */ std::set m_currentlyActiveIterations; + /** + * This map contains the filenames of those Iterations which were found + * on the file system upon opening the Series for reading in file-based + * encoding. It is only written to by readFileBased(). + * Other files that we create anew have names generated live by + * iterationFilename(), but files that existed previously might have + * different padding. + * This information is required for re-opening Iterations after closing. + * Since ADIOS2 has no read-write mode, this is important in our own + * READ_WRITE mode when re-opening a closed file in file-based encoding: + * A file that existed previously is re-opened in Read mode and will + * not support updating its contents. + * (Note that this is NOT a restriction of re-opening, this is + * fundamentally a restriction of R/W in ADIOS2. Files can be + * written XOR read.) + * A file that we created anew is re-opened in Append mode to continue + * writing data to it. Using `adios2.engine.parameters.FlattenSteps = + * "ON"` is recommended in this case. + */ + std::unordered_map m_iterationFilenames; /** * Needed if reading a single iteration of a file-based series. * Users may specify the concrete filename of one iteration instead of @@ -253,10 +267,10 @@ class Series : public Attributable friend class Iteration; friend class Writable; friend class ReadIterations; - friend class SeriesIterator; + friend class StatefulIterator; friend class internal::SeriesData; friend class internal::AttributableData; - friend class WriteIterations; + friend class StatefulSnapshotsContainer; public: explicit Series(); @@ -646,6 +660,64 @@ class Series : public Attributable */ ReadIterations readIterations(); + /** Parameter for Series::snapshots(), see there. + */ + enum class SnapshotWorkflow + { + RandomAccess, + Synchronous + }; + + /** @brief Preferred way to access Iterations/Snapshots. Single API for all + * workflows and access modi. + * + * Two fundamental workflows for Iteration/Snapshot processing exist: + * + * 1. Random-access workflow: Iterations/Snapshots are accessed + * independently from one another. Users must take care to open() + * and close() them as needed. + * More than one Iteration can be open at the same time. + * 2. Linear/Synchronous workflow: + * The (parallel) Series has one shared state. The effect is twofold: + * (a) Advancing one iterator, e.g. via `operator++()` will advance all + * other iterators as well. + * (b) Advancing an iterator is collective, all MPI ranks must + participate. + * The workflow is generally managed more automatically, + * `Iteration::open()` is not needed, `Iteration::close()` is + * recommended, but not needed. In READ_LINEAR mode, parsed Iteration + * data is deleted upon closing (and will be reparsed upon reopening) + * for a better support of datasets with many Snapshots/Iterations. This + * happens only if user code explicitly calls Iteration::close(). + * This mode generally brings better performance than random access mode + * due to the more restricted workflow. + * For accessing some kinds of Series, Synchronous access is even + * necessary, e.g. for Streaming workflows or variable-based encoding. + * (In future, random-access of Series with variable-based encoding will + * be possible under the condition that each Iteration/Snapshot has the + * same internal structure). + * + * As a rule of thumb, the synchronous workflow should be preferred as long + * as possible. The random-access workflow should be chosen when more + * flexible interaction with Snapshots is needed. + * + * Random-vs.-Synchronous access is determined automatically + * in READ workflows: Access::READ_LINEAR uses the synchronous workflow, + * while Access::READ_ONLY and Access::READ_WRITE use the random-access + * workflow. + * + * Conversely, the Access::CREATE and Access::APPEND access modes both + * resolve to random-access by default, but can be specified to use + * Synchronous workflow if needed. A shorthand for Synchronous workflows can + * be found with Series::writeIterations(). + * + * @param snapshot_workflow Specify the intended workflow + * in Access::CREATE and Access::APPEND. Leave unspecified in + * other access modes as those support only one workflow each. + */ + Snapshots + snapshots(std::optional snapshot_workflow = std::nullopt); + /** * @brief Parse the Series. * @@ -665,17 +737,9 @@ class Series : public Attributable /** * @brief Entry point to the writing end of the streaming API. * - * Creates and returns an instance of the WriteIterations class which is an - * intentionally restricted container of iterations that takes care of - * streaming semantics, e.g. ensuring that an iteration cannot be reopened - * once closed. - * For a less restrictive API in non-streaming situations, - * `Series::iterations` can be accessed directly. - * The created object is stored as member of the Series object, hence this - * method may be called as many times as a user wishes. - * There is only one shared iterator state per Series, even when calling - * this method twice. - * Look for the WriteIterations class for further documentation. + * Shorthand for `Series::snapshots()` for access types CREATE and APPEND + * called with parameter SnapshotWorkflow::Synchronous, i.e. for + * streaming-aware data producers. * * @return WriteIterations */ @@ -816,7 +880,11 @@ OPENPMD_private void flushMeshesPath(); void flushParticlesPath(); void flushRankTable(); - void readFileBased(); + /* Parameter `read_only_this_single_iteration` used for reopening an + * Iteration after closing it. + */ + void readFileBased( + std::optional read_only_this_single_iteration); void readOneIterationFileBased(std::string const &filePath); /** * Note on re-parsing of a Series: @@ -830,11 +898,13 @@ OPENPMD_private * If true, the error will always be re-thrown (useful when using * ReadIterations since those methods should be aware when the current step * is broken). + * Parameter `read_only_this_single_iteration` used for reopening an + * Iteration after closing it. */ - std::optional> readGorVBased( + std::vector readGorVBased( bool do_always_throw_errors, bool init, - std::set const &ignoreIterations = {}); + std::optional read_only_this_single_iteration); void readBase(); std::string iterationFilename(IterationIndex_t i); @@ -850,14 +920,14 @@ OPENPMD_private * parse state. */ IterationOpened - openIterationIfDirty(IterationIndex_t index, Iteration iteration); + openIterationIfDirty(IterationIndex_t index, Iteration &iteration); /* * Open an iteration. Ensures that the iteration's m_closed status * is set properly and that any files pertaining to the iteration * is opened. * Does not create files when called in CREATE mode. */ - void openIteration(IterationIndex_t index, Iteration iteration); + void openIteration(IterationIndex_t index, Iteration &iteration); /** * Find the given iteration in Series::iterations and return an iterator @@ -877,14 +947,12 @@ OPENPMD_private * based layout, it's the Series object. * @param it The iterator within Series::iterations pointing to that * iteration. - * @param iteration The actual Iteration object. * @return AdvanceStatus */ AdvanceStatus advance( AdvanceMode mode, internal::AttributableData &file, - iterations_iterator it, - Iteration &iteration); + iterations_iterator it); AdvanceStatus advance(AdvanceMode mode); @@ -909,12 +977,14 @@ OPENPMD_private AbstractIOHandler const *IOHandler() const; }; // Series +using SnapshotWorkflow = Series::SnapshotWorkflow; + namespace debug { void printDirty(Series const &); } } // namespace openPMD -// Make sure that this one is always included if Series.hpp is included, -// otherwise Series::readIterations() cannot be used +// Make sure that this legacy header is always included if Series.hpp is +// included, otherwise Series::readIterations() cannot be used #include "openPMD/ReadIterations.hpp" diff --git a/include/openPMD/Streaming.hpp b/include/openPMD/Streaming.hpp index 94854662fa..8d1a283761 100644 --- a/include/openPMD/Streaming.hpp +++ b/include/openPMD/Streaming.hpp @@ -45,6 +45,7 @@ enum class AdvanceMode : unsigned char enum class StepStatus : unsigned char { DuringStep, /* step is currently active */ + OutOfStep, /* steps used, but currently no step active */ NoStep /* no step is currently active */ }; } // namespace openPMD diff --git a/include/openPMD/WriteIterations.hpp b/include/openPMD/WriteIterations.hpp index fcf4a8fbfa..932a651f7f 100644 --- a/include/openPMD/WriteIterations.hpp +++ b/include/openPMD/WriteIterations.hpp @@ -1,109 +1,5 @@ -/* Copyright 2021 Franz Poeschel - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ #pragma once -#include "openPMD/Iteration.hpp" -#include "openPMD/backend/Container.hpp" +// legacy header -#include - -namespace openPMD -{ -class Series; - -/** Writing side of the streaming API. - * - * Create instance via Series::writeIterations(). - * For use via WriteIterations::operator[](). - * Designed to allow reading any kind of Series, streaming and non- - * streaming alike. Calling Iteration::close() manually before opening - * the next iteration is encouraged and will implicitly flush all - * deferred IO actions. Otherwise, Iteration::close() will be implicitly - * called upon SeriesIterator::operator++(), i.e. upon going to the next - * iteration in the foreach loop. - * - * Since this is designed for streaming mode, reopening an iteration is - * not possible once it has been closed. - * - */ - -namespace internal -{ - class SeriesData; -} - -/** - * @brief Writing side of the streaming API. - * - * Create instance via Series::writeIterations(). - * Restricted Container of Iterations, designed to allow reading any kind - * of Series, streaming and non-streaming alike. - * Calling Iteration::close() manually before opening the next iteration is - * encouraged and will implicitly flush all deferred IO actions. - * Otherwise, Iteration::close() will be implicitly called upon - * opening the next iteration or upon destruction. - * Since this is designed for streaming mode, reopening an iteration is - * not possible once it has been closed. - */ -class WriteIterations -{ - friend class Series; - friend class internal::SeriesData; - -private: - using IterationsContainer_t = - Container; - -public: - using key_type = IterationsContainer_t::key_type; - using mapped_type = IterationsContainer_t::mapped_type; - using value_type = IterationsContainer_t::value_type; - using reference = IterationsContainer_t::reference; - -private: - struct SharedResources - { - IterationsContainer_t iterations; - //! Index of the last opened iteration - std::optional currentlyOpen; - - SharedResources(IterationsContainer_t); - ~SharedResources(); - }; - - WriteIterations(IterationsContainer_t); - explicit WriteIterations() = default; - // std::optional so that a single instance is able to close this without - // needing to wait for all instances to deallocate - std::shared_ptr> shared; - - void close(); - -public: - mapped_type &operator[](key_type const &key); - mapped_type &operator[](key_type &&key); - - /** - * Return the iteration that is currently being written to, if it exists. - */ - std::optional currentIteration(); -}; -} // namespace openPMD +#include "openPMD/snapshots/Snapshots.hpp" diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 6ff5b85ed1..a77d8fe524 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -56,18 +56,19 @@ namespace internal class IterationData; class SeriesData; - class AttributableData + class SharedAttributableData { friend class openPMD::Attributable; public: - AttributableData(); - AttributableData(AttributableData const &) = delete; - AttributableData(AttributableData &&) = delete; - virtual ~AttributableData() = default; + SharedAttributableData(AttributableData *); + SharedAttributableData(SharedAttributableData const &) = delete; + SharedAttributableData(SharedAttributableData &&) = delete; + virtual ~SharedAttributableData() = default; - AttributableData &operator=(AttributableData const &) = delete; - AttributableData &operator=(AttributableData &&) = delete; + SharedAttributableData & + operator=(SharedAttributableData const &) = delete; + SharedAttributableData &operator=(SharedAttributableData &&) = delete; using A_MAP = std::map; /** @@ -77,6 +78,45 @@ namespace internal */ Writable m_writable; + /** + * The attributes defined by this Attributable. + */ + A_MAP m_attributes; + }; + + /* + * This is essentially a two-level pointer. + * + * 1. level: Our public API hands out handles to users that are (shared) + * pointers to an internal object (PIMPL). + * 2. level: Multiple internal objects might refer to the same item in an + * openPMD file, e.g. to the same backend object. + * So, the internal object for an Attributable is a shared pointer to the + * unique object identifying this item. + * + * Such sharing occurs in the CustomHierarchy class where multiple + * containers refer to the same group in the openPMD hierarchy + * (container of groups, of meshes, of particle species, of datasets). + * This might also become relevant for links as in HDF5 if we choose to + * implement them. + */ + + class AttributableData : public std::shared_ptr + { + friend class openPMD::Attributable; + + using SharedData_t = std::shared_ptr; + + public: + AttributableData(); + AttributableData(SharedAttributableData *); + AttributableData(AttributableData const &) = delete; + AttributableData(AttributableData &&) = delete; + virtual ~AttributableData() = default; + + AttributableData &operator=(AttributableData const &) = delete; + AttributableData &operator=(AttributableData &&) = delete; + template T asInternalCopyOf() { @@ -112,12 +152,6 @@ namespace internal std::shared_ptr(self, [](auto const *) {})); return res; } - - private: - /** - * The attributes defined by this Attributable. - */ - A_MAP m_attributes; }; template @@ -169,11 +203,11 @@ class Attributable friend class Iteration; friend class Series; friend class Writable; - friend class WriteIterations; friend class internal::RecordComponentData; friend void debug::printDirty(Series const &); template friend T &internal::makeOwning(T &self, Series); + friend class StatefulSnapshotsContainer; protected: // tag for internal constructor @@ -185,7 +219,7 @@ class Attributable public: Attributable(); - Attributable(NoInit); + Attributable(NoInit) noexcept; virtual ~Attributable() = default; @@ -431,7 +465,7 @@ OPENPMD_protected } AbstractIOHandler const *IOHandler() const { - auto &opt = m_attri->m_writable.IOHandler; + auto &opt = writable().IOHandler; if (!opt || !opt->has_value()) { return nullptr; @@ -440,19 +474,19 @@ OPENPMD_protected } Writable *&parent() { - return m_attri->m_writable.parent; + return writable().parent; } Writable const *parent() const { - return m_attri->m_writable.parent; + return writable().parent; } Writable &writable() { - return m_attri->m_writable; + return (*m_attri)->m_writable; } Writable const &writable() const { - return m_attri->m_writable; + return (*m_attri)->m_writable; } inline void setData(std::shared_ptr attri) @@ -460,13 +494,13 @@ OPENPMD_protected m_attri = std::move(attri); } - inline internal::AttributableData &get() + inline internal::SharedAttributableData &get() { - return *m_attri; + return **m_attri; } - inline internal::AttributableData const &get() const + inline internal::SharedAttributableData const &get() const { - return *m_attri; + return **m_attri; } bool dirty() const diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index cee50f9baf..8dda5b992b 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -113,7 +113,7 @@ class Container : virtual public Attributable friend class Series; template friend class internal::EraseStaleEntries; - friend class SeriesIterator; + friend class StatefulIterator; protected: using ContainerData = internal::ContainerData; @@ -489,7 +489,7 @@ OPENPMD_protected m_containerData = other.m_containerData; } - Container(Container &&other) : Attributable(NoInit()) + Container(Container &&other) noexcept : Attributable(NoInit()) { if (other.m_attri) { @@ -505,7 +505,7 @@ OPENPMD_protected return *this; } - Container &operator=(Container &&other) + Container &operator=(Container &&other) noexcept { if (other.m_attri) { diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 25a2154665..be36f47758 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -48,6 +48,7 @@ class Series; namespace internal { + class SharedAttributableData; class AttributableData; class SeriesData; } // namespace internal @@ -73,6 +74,7 @@ namespace debug */ class Writable final { + friend class internal::SharedAttributableData; friend class internal::AttributableData; friend class internal::SeriesData; friend class Attributable; @@ -142,6 +144,11 @@ OPENPMD_private */ std::shared_ptr>> IOHandler = nullptr; + /* + * Link to the containing Attributable. + * If multiple Attributables share the same Writable, then the creating one. + * (See SharedAttributableData) + */ internal::AttributableData *attributable = nullptr; Writable *parent = nullptr; diff --git a/include/openPMD/openPMD.hpp b/include/openPMD/openPMD.hpp index 08853b4d1e..0773243b66 100644 --- a/include/openPMD/openPMD.hpp +++ b/include/openPMD/openPMD.hpp @@ -34,12 +34,11 @@ namespace openPMD #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" #include "openPMD/ParticleSpecies.hpp" -#include "openPMD/ReadIterations.hpp" #include "openPMD/Record.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" #include "openPMD/UnitDimension.hpp" -#include "openPMD/WriteIterations.hpp" +#include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Attribute.hpp" diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp new file mode 100644 index 0000000000..8a0189e80e --- /dev/null +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -0,0 +1,133 @@ +#pragma once + +#include "openPMD/snapshots/ContainerTraits.hpp" +#include "openPMD/snapshots/RandomAccessIterator.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" + +#include + +/* + * Private header, not included in user code. + */ + +namespace openPMD +{ +class StatefulSnapshotsContainer : public AbstractSnapshotsContainer +{ +private: + friend class Series; + + struct Members + { + /* + * Consider the following user code: + * > auto iterations = series.snapshots(); + * > for (auto & iteration : iterations) { ... } + * + * Here, only the for loop should actually wait for Iteration data. For + * ensuring that Iterations are not waited for too early, the + * initialization procedures are stored as a `std::function` in here and + * only called upon the right moment. Until then, m_bufferedIterator + * stays empty. + * Compare the implementation of Series::snapshots(). In there, m_begin + * is defined either by make_writing_stateful_iterator or + * make_reading_stateful_iterator. + * The iterator is resolved upon calling get() below. + */ + std::function m_begin; + std::optional m_bufferedIterator = std::nullopt; + }; + Members members; + + StatefulSnapshotsContainer(std::function begin); + + auto get() -> StatefulIterator *; + auto get() const -> StatefulIterator const *; + +public: + ~StatefulSnapshotsContainer() override; + + StatefulSnapshotsContainer(StatefulSnapshotsContainer const &other); + StatefulSnapshotsContainer(StatefulSnapshotsContainer &&other) noexcept( + noexcept(Members(std::declval()))); + + StatefulSnapshotsContainer & + operator=(StatefulSnapshotsContainer const &other); + StatefulSnapshotsContainer & + operator=(StatefulSnapshotsContainer &&other) noexcept(noexcept( + std::declval().operator=(std::declval()))); + + using AbstractSnapshotsContainer::currentIteration; + auto currentIteration() const -> std::optional override; + + auto begin() -> iterator override; + auto end() -> iterator override; + auto begin() const -> const_iterator override; + auto end() const -> const_iterator override; + auto rbegin() -> iterator override; + auto rend() -> iterator override; + auto rbegin() const -> const_iterator override; + auto rend() const -> const_iterator override; + + auto empty() const -> bool override; + auto size() const -> size_t override; + + auto at(key_type const &key) const -> mapped_type const & override; + auto at(key_type const &key) -> mapped_type & override; + + auto operator[](key_type const &key) -> mapped_type & override; + + auto clear() -> void override; + + auto find(key_type const &key) -> iterator override; + auto find(key_type const &key) const -> const_iterator override; + + auto contains(key_type const &key) const -> bool override; +}; + +class RandomAccessIteratorContainer : public AbstractSnapshotsContainer +{ +private: + friend class Series; + Container m_cont; + RandomAccessIteratorContainer(Container cont); + +public: + ~RandomAccessIteratorContainer() override; + + RandomAccessIteratorContainer(RandomAccessIteratorContainer const &other); + RandomAccessIteratorContainer( + RandomAccessIteratorContainer &&other) noexcept; + + RandomAccessIteratorContainer & + operator=(RandomAccessIteratorContainer const &other); + RandomAccessIteratorContainer & + operator=(RandomAccessIteratorContainer &&other) noexcept; + + using AbstractSnapshotsContainer::currentIteration; + auto currentIteration() const -> std::optional override; + + auto begin() -> iterator override; + auto end() -> iterator override; + auto begin() const -> const_iterator override; + auto end() const -> const_iterator override; + auto rbegin() -> iterator override; + auto rend() -> iterator override; + auto rbegin() const -> const_iterator override; + auto rend() const -> const_iterator override; + + auto empty() const -> bool override; + auto size() const -> size_t override; + + using AbstractSnapshotsContainer::at; + auto at(key_type const &key) const -> mapped_type const & override; + auto operator[](key_type const &key) -> mapped_type & override; + + auto clear() -> void override; + + auto find(key_type const &key) -> iterator override; + auto find(key_type const &key) const -> const_iterator override; + + auto contains(key_type const &key) const -> bool override; +}; +} // namespace openPMD diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp new file mode 100644 index 0000000000..4bfe73b518 --- /dev/null +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -0,0 +1,113 @@ +#pragma once + +#include "openPMD/Iteration.hpp" +#include "openPMD/snapshots/IteratorTraits.hpp" + +/* Public header due to use of AbstractSnapshotsContainer and its iterator type + * OpaqueSeriesIterator in Snapshots class header. No direct user interaction + * required with this header. + */ + +namespace openPMD +{ +/** Counterpart to Snapshots class: + * Iterator type that can wrap different implementations internally. + */ +template +class OpaqueSeriesIterator + : public AbstractSeriesIterator< + OpaqueSeriesIterator, + value_type_in> +{ +private: + friend class Series; + friend class StatefulSnapshotsContainer; + friend class RandomAccessIteratorContainer; + using parent_t = + AbstractSeriesIterator; + // no shared_ptr since copied iterators should not share state + std::unique_ptr> m_internal_iterator; + +public: + OpaqueSeriesIterator(std::unique_ptr> + internal_iterator); + + OpaqueSeriesIterator(OpaqueSeriesIterator const &other); + OpaqueSeriesIterator(OpaqueSeriesIterator &&other) noexcept; + OpaqueSeriesIterator &operator=(OpaqueSeriesIterator const &other); + OpaqueSeriesIterator &operator=(OpaqueSeriesIterator &&other) noexcept; + + ~OpaqueSeriesIterator(); + + // dereference + using value_type = value_type_in; + + auto operator*() -> value_type &; + auto operator*() const -> value_type const &; + + // increment/decrement + OpaqueSeriesIterator &operator++(); + /** Not implemented for synchronous workflow: + * Reverse iteration not possible. + */ + OpaqueSeriesIterator &operator--(); + /** Not implemented for synchronous workflow: + * Post increment not possible. + */ + OpaqueSeriesIterator operator++(int); + /** Not implemented for synchronous workflow: + * Reverse iteration not possible. + */ + OpaqueSeriesIterator operator--(int); + + // comparison + bool operator==(OpaqueSeriesIterator const &other) const; +}; + +// Internal interface used by Snapshots class for interacting with containers. +// This needs to be in a public header since the type definition is used in +// private members of the Snapshots class which itself is a public class. +class AbstractSnapshotsContainer +{ +public: + using key_type = Iteration::IterationIndex_t; + using value_type = Container::value_type; + using mapped_type = Iteration; + using iterator = OpaqueSeriesIterator; + using const_iterator = OpaqueSeriesIterator; + // since AbstractSnapshotsContainer abstracts away the specific mode of + // iteration, these are the same type + using reverse_iterator = OpaqueSeriesIterator; + using const_reverse_iterator = OpaqueSeriesIterator; + + virtual ~AbstractSnapshotsContainer() = 0; + + virtual auto currentIteration() -> std::optional; + virtual auto currentIteration() const + -> std::optional = 0; + + virtual auto begin() -> iterator = 0; + virtual auto begin() const -> const_iterator = 0; + virtual auto end() -> iterator = 0; + virtual auto end() const -> const_iterator = 0; + virtual auto rbegin() -> reverse_iterator = 0; + virtual auto rbegin() const -> const_reverse_iterator = 0; + virtual auto rend() -> reverse_iterator = 0; + virtual auto rend() const -> const_reverse_iterator = 0; + + virtual auto empty() const -> bool = 0; + virtual auto size() const -> size_t = 0; + + virtual auto at(key_type const &key) const -> mapped_type const & = 0; + virtual auto at(key_type const &key) -> mapped_type &; + + virtual auto operator[](key_type const &key) -> mapped_type & = 0; + + virtual auto clear() -> void = 0; + + virtual auto find(key_type const &key) -> iterator = 0; + virtual auto find(key_type const &key) const -> const_iterator = 0; + + virtual auto contains(key_type const &key) const -> bool = 0; +}; +} // namespace openPMD diff --git a/include/openPMD/snapshots/IteratorHelpers.hpp b/include/openPMD/snapshots/IteratorHelpers.hpp new file mode 100644 index 0000000000..cc9c737c78 --- /dev/null +++ b/include/openPMD/snapshots/IteratorHelpers.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "openPMD/Iteration.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" + +namespace openPMD +{ +using value_type = + Container::value_type; +auto stateful_to_opaque(StatefulIterator const &it) + -> OpaqueSeriesIterator; +} // namespace openPMD diff --git a/include/openPMD/snapshots/IteratorTraits.hpp b/include/openPMD/snapshots/IteratorTraits.hpp new file mode 100644 index 0000000000..f44f3c25b1 --- /dev/null +++ b/include/openPMD/snapshots/IteratorTraits.hpp @@ -0,0 +1,161 @@ +/* Copyright 2024 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/Iteration.hpp" +#include "openPMD/backend/Writable.hpp" + +#include + +/* + * Public header due to use in OpaqueSeriesIterator type which is the public + * iterator type of Snapshots class. + */ + +namespace openPMD +{ +/* + * Abstract class that can be used as an abstract interface to an opaque + * iterator implementation + * + * This has renamed operator names for two reasons: + * + * 1. Name shadowing of default implementations is too finnicky. + * 2. The return type for the actual operators should be a reference to the + * child class, but for an Interface we need a common return type. + * + * For returning a reference to the child class, we need a CRT-style template to + * know the type. That does not work for an interface. As a result, this generic + * Iterator interface is split in two parts, class DynamicSeriesIterator which + * can be used generically without specifying the child class. All methods that + * need to know the child class type are put into the CRT-style class template + * AbstractSeriesIterator below. + */ +template < + typename value_type = + Container::value_type> +class DynamicSeriesIterator +{ +public: + using difference_type = Iteration::IterationIndex_t; + virtual ~DynamicSeriesIterator() = 0; + +protected: + template + friend class OpaqueSeriesIterator; + + // dereference + virtual value_type const &dereference_operator() const = 0; + virtual value_type &dereference_operator(); + + // increment/decrement + virtual DynamicSeriesIterator &increment_operator() = 0; + virtual DynamicSeriesIterator &decrement_operator() = 0; + + // comparison + virtual bool equality_operator(DynamicSeriesIterator const &) const = 0; + + virtual std::unique_ptr clone() const = 0; +}; +/* + * Class template with default method implementations for iterators. + * Complementary class to above class DynamicSeriesIterator, containing those + * methods that have the specific Iterator type in their type specification. See + * the documentation for DynamicSeriesIterator for more details. + * No virtual classes since there is no use. + * Commented methods must be implemented by child classes. + * Implement as `class MyIterator : public AbstractSeriesIterator` + * Use `using AbstractSeriesIterator::operator-` to pull default + * implementations. + */ +template < + typename ChildClass, + typename value_type_in = typename ChildClass::value_type> +class AbstractSeriesIterator : public DynamicSeriesIterator +{ +public: + using difference_type = Iteration::IterationIndex_t; + using value_type = value_type_in; + + ~AbstractSeriesIterator() override; + + /* + * Default definitions that can be pulled in implementing child classes with + * `using` declarations. Does not work for overloaded methods due to + * compiler bugs in ICPC, hence e.g. `default_increment_operator` instead of + * `operator++`. + */ + + // dereference + // value_type const &operator*() const = 0; + // value_type &operator*() = 0; + auto operator->() const -> value_type const *; + auto operator->() -> value_type *; + + // increment/decrement + // ChildClass &operator++(); + // ChildClass &operator--(); + // ChildClass &operator++(int); + // ChildClass &operator--(int); + auto default_increment_operator(int) -> ChildClass; + auto default_decrement_operator(int) -> ChildClass; + + // comparison + // bool operator==(ChildClass const &) const = 0; + bool operator!=(ChildClass const &) const; + + /************* + * overrides * + *************/ + +protected: + using parent_t = DynamicSeriesIterator; + // dereference + using parent_t::dereference_operator; + auto dereference_operator() const -> value_type const & override; + + // increment/decrement + auto increment_operator() -> parent_t & override; + auto decrement_operator() -> parent_t & override; + + // comparison + auto equality_operator(parent_t const &) const -> bool override; + + auto clone() const -> std::unique_ptr override; + +private: + ChildClass *this_child(); + ChildClass const *this_child() const; +}; + +template +ChildClass operator+( + Iteration::IterationIndex_t, AbstractSeriesIterator const &); +template +ChildClass +operator+(Iteration::IterationIndex_t, AbstractSeriesIterator &); +template +ChildClass operator-( + Iteration::IterationIndex_t, AbstractSeriesIterator const &); +template +ChildClass +operator-(Iteration::IterationIndex_t, AbstractSeriesIterator &); +} // namespace openPMD diff --git a/include/openPMD/snapshots/RandomAccessIterator.hpp b/include/openPMD/snapshots/RandomAccessIterator.hpp new file mode 100644 index 0000000000..1487d64aa7 --- /dev/null +++ b/include/openPMD/snapshots/RandomAccessIterator.hpp @@ -0,0 +1,85 @@ +/* Copyright 2021 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/Iteration.hpp" +#include "openPMD/snapshots/IteratorTraits.hpp" + +#include + +/* + * Private header not included in user code. + * Implements the Iterator interface for the random-access workflow. + */ + +namespace openPMD +{ +namespace detail +{ + template + using iterator_to_value_type = + // do NOT remove const from type + std::remove_reference_t())>; +} + +template +class RandomAccessIterator + : public AbstractSeriesIterator< + RandomAccessIterator, + detail::iterator_to_value_type> +{ +private: + friend class RandomAccessIteratorContainer; + using parent_t = AbstractSeriesIterator< + RandomAccessIterator, + detail::iterator_to_value_type>; + + RandomAccessIterator(iterator_t it); + + /* Internal iterator */ + iterator_t m_it; + +public: + using typename parent_t::value_type; + + ~RandomAccessIterator() override; + + RandomAccessIterator(RandomAccessIterator const &other); + RandomAccessIterator(RandomAccessIterator &&other) noexcept( + noexcept(iterator_t(std::declval()))); + + RandomAccessIterator &operator=(RandomAccessIterator const &other); + RandomAccessIterator & + operator=(RandomAccessIterator &&other) noexcept(noexcept( + std::declval().operator=(std::declval()))); + + auto operator*() -> value_type &; + auto operator*() const -> value_type const &; + + auto operator++() -> RandomAccessIterator &; + auto operator--() -> RandomAccessIterator &; + auto operator++(int) -> RandomAccessIterator; + auto operator--(int) -> RandomAccessIterator; + + using parent_t::operator!=; + bool operator==(RandomAccessIterator const &other) const; +}; +} // namespace openPMD diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp new file mode 100644 index 0000000000..7a75780dfe --- /dev/null +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -0,0 +1,163 @@ +/* Copyright 2024 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/Iteration.hpp" +#include "openPMD/snapshots/ContainerTraits.hpp" +#include +#include +#include + +namespace openPMD +{ +/** Entry point for accessing Snapshots/Iterations. Common API for different + * Snapshot accessing workflows. + * + * Refer to Series::snapshots() for a detailed description of different + * workflows and their applications. + * The API generally follows the Container/map conventions, though especially + * the synchronous workflow does not implement all of the API calls. + * + * Since the synchronous workflow is tailored to support use cases from ADIOS2 + * which do not support going back to the start (IO steps, streaming), the + * Container semantics are based on the remaining Iterations that have not yet + * been seen. Especially does this mean that Snapshots::begin() does not reset + * the Iterator back to the start, but will return the current state. There is + * restricted support for going back to past Iterations by reopening them under + * the condition that no previous ADIOS2 step needs to be activated. Note that + * an Iteration handle goes invalid after closing it, a new Iteration handle is + * acquired by Snapshots::operator[](). + */ +class Snapshots +{ +private: + friend class Series; + + std::shared_ptr m_snapshots; + + Snapshots(std::shared_ptr snapshots); + + inline auto get() const -> AbstractSnapshotsContainer const &; + inline auto get() -> AbstractSnapshotsContainer &; + +public: + using key_type = AbstractSnapshotsContainer::key_type; + using value_type = AbstractSnapshotsContainer::value_type; + using mapped_type = AbstractSnapshotsContainer::mapped_type; + using iterator = AbstractSnapshotsContainer::iterator; + using const_iterator = AbstractSnapshotsContainer::const_iterator; + // since AbstractSnapshotsContainer abstracts away the specific mode of + // iteration, these are the same type + using reverse_iterator = AbstractSnapshotsContainer::reverse_iterator; + using const_reverse_iterator = + AbstractSnapshotsContainer::const_reverse_iterator; + + /** @brief The currently active Iteration. + * + * Mostly useful for the synchronous workflow, since that has a notion of + * global state where a specific Iteration is currently active (or no + * Iteration, in which case a nullopt is returned). For random-access + * workflow, the first Iteration in the Series is returned. + */ + auto currentIteration() -> std::optional; + auto currentIteration() const -> std::optional; + + auto begin() -> iterator; + auto end() -> iterator; + /** Not implemented for synchronous workflow: Const iteration not possible. + */ + auto begin() const -> const_iterator; + /** Not implemented for synchronous workflow: Const iteration not possible. + */ + auto end() const -> const_iterator; + /** Not implemented for synchronous workflow: + * Reverse iteration not possible. + */ + auto rbegin() -> reverse_iterator; + /** Not implemented for synchronous workflow: + * Reverse iteration not possible. + */ + auto rend() -> reverse_iterator; + /** Not implemented for synchronous workflow: + * Const reverse iteration not possible. + */ + auto rbegin() const -> const_reverse_iterator; + /** Not implemented for synchronous workflow: + * Const reverse iteration not possible. + */ + auto rend() const -> const_reverse_iterator; + + /** In synchronous workflow, this tells if there are remaining Iterations or + * not. + */ + auto empty() const -> bool; + /** Not implemented in synchronous workflow due to unclear semantics (past + * Iterations should not be considered, future Iterations are not yet + * known). + */ + auto size() const -> size_t; + + /** Select an Iteration within the current IO step. + */ + auto at(key_type const &key) const -> mapped_type const &; + /** Select an Iteration within the current IO step. + */ + auto at(key_type const &key) -> mapped_type &; + + auto operator[](key_type const &key) -> mapped_type &; + + /** Not implmented in synchronous workflow. + */ + auto clear() -> void; + + // insert + // swap + + /** Not implmented in synchronous workflow. + * + * (In future: Might be implemented in terms of searching within the current + * IO step) + */ + auto find(key_type const &key) -> iterator; + /** Not implmented in synchronous workflow. + * + * (In future: Might be implemented in terms of checking the current + * Iteration against the searched key and returning end() if it doesn't + * match. Anything else is not possible since the shared Iterator would have + * to be modified.) + */ + auto find(key_type const &key) const -> const_iterator; + + /** Implemented in terms of contains(), see there. + */ + auto count(key_type const &key) const -> size_t; + + /** Not implmented in synchronous workflow. + */ + auto contains(key_type const &key) const -> bool; + + // erase + // emplace +}; + +// backwards compatibility +using WriteIterations = Snapshots; +} // namespace openPMD diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp new file mode 100644 index 0000000000..26ebc4075e --- /dev/null +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -0,0 +1,437 @@ +/* Copyright 2021 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/Error.hpp" +#include "openPMD/Iteration.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/Streaming.hpp" +#include "openPMD/auxiliary/Variant.hpp" +#include "openPMD/backend/ParsePreference.hpp" +#include "openPMD/snapshots/IteratorTraits.hpp" + +#include +#include +#include +#include +#include +#include +#include + +/* + * Private header not included in user code. + * Implements the Iterator interface for the stateful/synchronous workflow. + */ + +namespace openPMD +{ +namespace internal +{ + class SeriesData; +} + +namespace detail +{ + /* The iterator status is either of the following: + */ + namespace step_status_types + { + /* No step was opened yet, the Series was just opened. + */ + struct Before_t + {}; + /* A step is currently active + */ + struct During_t + { + // The index of the current Step. + size_t step_count; + // The current Iteration within the Step. + // Empty optional indicates that no Iteration is left in the current + // step for processing, i.e. a new step must be opened or the Series + // is over. + std::optional iteration_idx; + // Iteration indexes that are accessible within the current step. + // These are not modified when closing an Iteration as long as the + // current IO step stays active. + std::vector + available_iterations_in_step; + + During_t( + size_t step_count, + std::optional iteration_idx, + std::vector + available_iterations_in_step); + }; + /* No further data available in the Series. + */ + struct After_t + {}; + } // namespace step_status_types + + /* This class unifies the current step status as described above into a + * std::variant with some helper functions. + */ + struct CurrentStep + : std::variant< + step_status_types::Before_t, + step_status_types::During_t, + step_status_types::After_t> + { + using Before_t = step_status_types::Before_t; + constexpr static Before_t Before{}; + using During_t = step_status_types::During_t; + using After_t = step_status_types::After_t; + constexpr static After_t After{}; + + using variant_t = std::variant< + step_status_types::Before_t, + step_status_types::During_t, + step_status_types::After_t>; + + using variant_t::operator=; + + template + auto get_variant() -> std::optional; + template + auto get_variant() const -> std::optional; + + auto get_iteration_index() const + -> std::optional; + + /* Passed as first param of create_new lambda in map_during_t, so the + * lambda can make an appropriate case distinction. + */ + enum class AtTheEdge : bool + { + Begin, + End + }; + + /* + * Helper for a common way to access the underlying variant. + * `map` has type `auto (During_t &) -> void`, i.e. it can modify the + * `During_t` struct if the variant holds it. In other cases, + * `create_new` is called, it has the type + * `auto (AtTheEdge) -> std::optional`. + * `AtTheEdge` is used for specifying if the variant status is Begin or + * End. If the returned optional contains a value, that value is swapped + * with the current variant. + */ + template + void map_during_t(F &&map, G &&create_new); + + /* + * Overload where `create_new` is a no-op. + */ + template + void map_during_t(F &&map); + + // casts needed because of + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90943 + inline auto as_base() const -> variant_t const & + { + return *this; + } + inline auto as_base() -> variant_t & + { + return *this; + } + }; + + /* + * Types for telling the Iterator where to go next. + */ + namespace seek_types + { + /* Just give me the next Iteration */ + struct Next_t + {}; + /* Give me some specific Iteration */ + struct Seek_Iteration_t + { + Iteration::IterationIndex_t iteration_idx; + }; + } // namespace seek_types + + struct Seek : std::variant + { + using Next_t = seek_types::Next_t; + using Seek_Iteration_t = seek_types::Seek_Iteration_t; + + static constexpr Next_t const Next{}; + + using variant_t = + std::variant; + // casts needed because of + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90943 + inline auto as_base() const -> variant_t const & + { + return *this; + } + inline auto as_base() -> variant_t & + { + return *this; + } + }; +} // namespace detail + +/** Based on the logic of the former class ReadIterations, integrating into + * itself the logic of former WriteIterations. + */ +class StatefulIterator + : public AbstractSeriesIterator< + StatefulIterator, + Container::value_type> +{ + friend class StatefulSnapshotsContainer; + friend class Series; + friend class internal::SeriesData; + + using iteration_index_t = IndexedIteration::index_t; + + using CurrentStep = detail::CurrentStep; + + struct SharedData + { + SharedData() = default; + SharedData(SharedData const &) = delete; + SharedData(SharedData &&) = delete; + SharedData &operator=(SharedData const &) = delete; + SharedData &operator=(SharedData &&) = delete; + + ~SharedData(); + + using step_index = size_t; + + /* + * This must be a non-owning internal handle to break reference cycles. + * A non-owning handle is fine due to the usual semantics for iterator + * invalidation. + */ + Series series; + /* + * No step opened yet, so initialize this with CurrentStep::Before. Look + * to the documentation of Before_t, During_t, After_t and CurrentStep + * classes above for more info. + */ + CurrentStep currentStep = {CurrentStep::Before}; + /* + * Stores the parse preference optionally passed in the constructor. + * Decides if IO step logic is actually used. + */ + std::optional parsePreference; + /* + * Store which Iterations we already saw and in which IO step we did. + * Currently used for eliminating repetitions when (e.g. due to + * checkpoint-restart workflows) Iterations repeat in different steps. + * + * Possible future uses: + * + * 1. Support jumping back to a previous step in order to reopen an + * Iteration previously seen. (Would require reopening files in + * ADIOS2, but so be it.) + * 2. Pre-parsing a variable-based file for repeating Iterations and + * eliminating the earlier instances of repeated Iterations (instead + * of the later instances as is done now). + */ + std::unordered_map seen_iterations; + + /* + * This returns the current value of `During_t::iteration_idx` if that + * exists. + */ + auto currentIteration() const + -> std::optional; + }; + + /* + * The shared pointer is never empty, + * emptiness is indicated by std::optional. + */ + std::shared_ptr> m_data = + std::make_shared>(std::nullopt); + + auto get() -> SharedData &; + auto get() const -> SharedData const &; + + using parent_t = AbstractSeriesIterator< + StatefulIterator, + Container::value_type>; + +public: + using value_type = + typename Container::value_type; + using typename parent_t ::difference_type; + using Seek = detail::Seek; + + //! construct the end() iterator + explicit StatefulIterator(); + ~StatefulIterator() override; + + StatefulIterator(StatefulIterator const &other); + StatefulIterator(StatefulIterator &&other) noexcept; + + StatefulIterator &operator=(StatefulIterator const &other); + StatefulIterator &operator=(StatefulIterator &&other) noexcept; + + class tag_write_t + {}; + static constexpr tag_write_t const tag_write{}; + class tag_read_t + {}; + static constexpr tag_read_t const tag_read{}; + + StatefulIterator( + tag_read_t, + Series const &, + std::optional const &parsePreference); + + StatefulIterator(tag_write_t, Series const &); + + // dereference + auto operator*() -> value_type &; + auto operator*() const -> value_type const &; + + // increment/decrement + auto operator++() -> StatefulIterator &; + auto operator--() -> StatefulIterator &; + auto operator--(int) -> StatefulIterator; + auto operator++(int) -> StatefulIterator; + + // comparison + auto operator-(StatefulIterator const &) const -> difference_type; + bool operator==(StatefulIterator const &other) const; + auto operator<(StatefulIterator const &) const -> bool; + + static auto end() -> StatefulIterator; + /* + * This is considered an end iterator if: + * + * 1. The iterator has no state at all + * (generic statically created end iterator) + * 2. The state is During_t with no iteration index + * (finished reading iterations in a randomly-accessible Series) + * 3. The state is After_t + * (closed the last step in a step-wise Series) + */ + auto is_end() const -> bool; + + operator bool() const; + + /* + * Try moving this Iterator to the location specified by Seek, i.e.: + * + * 1. Either the next available Iteration + * 2. Or a specific Iteration specified by an index. + * + * A new step will be opened for this purpose if needed. + */ + auto seek(Seek const &) -> StatefulIterator &; + +private: + std::optional nextIterationInStep(); + std::optional skipToIterationInStep(iteration_index_t); + + /* + * When a step cannot successfully be opened, the method nextStep() calls + * itself again recursively. + * (Recursion massively simplifies the logic here, and it only happens + * in case of error.) + * After successfully beginning a step, this methods needs to remember, how + * many broken steps have been skipped. In case the Series does not use + * the /data/snapshot attribute, this helps figuring out which iteration + * is now active. Hence, recursion_depth. + */ + std::optional nextStep(size_t recursion_depth); + + std::optional loopBody(Seek const &); + + void initIteratorFilebased(); + + /* + * Called when an Iteration was just opened but entirely fails parsing. + */ + void deactivateDeadIteration(iteration_index_t); + + void initSeriesInLinearReadMode(); + + void close(); + + /* When not using IO steps, the status should not be set to After_t, but be + * kept as During_t. This way, Iterations can still be opened without the + * Iterator thinking it's from a past step. + */ + enum class TypeOfEndIterator : char + { + NoMoreSteps, + NoMoreIterationsInStep + }; + auto turn_into_end_iterator(TypeOfEndIterator) -> void; + auto assert_end_iterator() const -> void; + + auto resetCurrentIterationToBegin( + size_t num_skipped_iterations, + std::vector current_iterations) -> void; + auto peekCurrentlyOpenIteration() const + -> std::optional; + auto peekCurrentlyOpenIteration() -> std::optional; + + auto reparse_possibly_deleted_iteration(iteration_index_t) -> void; +}; +} // namespace openPMD + +// Template definitions + +namespace openPMD::detail +{ +template +void CurrentStep::map_during_t(F &&map, G &&create_new) +{ + std::visit( + auxiliary::overloaded{ + [&](During_t &during) { std::forward(map)(during); }, + [&](Before_t const &) { + std::optional res = + std::forward(create_new)(AtTheEdge::Begin); + if (res.has_value()) + { + this->swap(*res); + } + }, + [&](After_t const &) { + std::optional res = + std::forward(create_new)(AtTheEdge::End); + if (res.has_value()) + { + this->swap(*res); + } + }}, + this->as_base()); +} + +template +void CurrentStep::map_during_t(F &&map) +{ + map_during_t( + std::forward(map), [](auto const &) { return std::nullopt; }); +} +} // namespace openPMD::detail diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index ee0c1a9062..82dd33f70b 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -23,6 +23,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/StringManip.hpp" @@ -159,7 +160,10 @@ void BufferedUniquePtrPut::run(ADIOS2File &ba) switchAdios2VariableType(dtype, *this, ba); } -ADIOS2File::ADIOS2File(ADIOS2IOHandlerImpl &impl, InvalidatableFile file) +ADIOS2File::ADIOS2File( + ADIOS2IOHandlerImpl &impl, + InvalidatableFile file, + adios_defs::OpenFileAs openFileAs) : m_file(impl.fullPath(std::move(file))) , m_ADIOS(impl.m_ADIOS) , m_impl(&impl) @@ -167,7 +171,7 @@ ADIOS2File::ADIOS2File(ADIOS2IOHandlerImpl &impl, InvalidatableFile file) // Declaring these members in the constructor body to avoid // initialization order hazards. Need the IO_ prefix since in some // situation there seems to be trouble with number-only IO names - m_mode = impl.adios2AccessMode(m_file); + m_mode = impl.adios2AccessMode(m_file, openFileAs); create_IO(); if (!m_IO) { diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 855958c888..4297cbe97a 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -37,6 +37,7 @@ #include #include // std::tolower +#include #include #include #include @@ -599,7 +600,8 @@ to drastic performance issues, no matter if I/O steps are used or not. * If using I/O steps: Each step will add new variables and attributes instead of reusing those from earlier steps. ADIOS2 is not optimized for this and especially the BP5 engine will show a quadratic increase in metadata size - as the number of steps increase. + as the number of steps increase. Consider moving to the BP4 engine if you want + to keep using group-based encoding. We advise you to pick either file-based encoding or variable-based encoding (variable-based encoding is not yet feature-complete in the openPMD-api). For more details, refer to @@ -649,7 +651,7 @@ void ADIOS2IOHandlerImpl::createFile( // enforce opening the file // lazy opening is deathly in parallel situations auto &fileData = - getFileData(shared_name, IfFileNotOpen::OpenImplicitly); + getFileData(shared_name, IfFileNotOpen::CreateImplicitly); if (!printedWarningsAlready.noGroupBased && m_writeAttributesFromThisRank && @@ -658,15 +660,20 @@ void ADIOS2IOHandlerImpl::createFile( // For a peaceful phase-out of group-based encoding in ADIOS2, // print this warning only in the new layout (with group table) if (m_useGroupTable.value_or(UseGroupTable::No) == - UseGroupTable::Yes) + UseGroupTable::Yes && + (m_engineType == "bp5" +#if openPMD_HAS_ADIOS_2_9 + || (m_engineType == "file" || m_engineType == "filestream" || + m_engineType == "bp") +#endif + )) { std::cerr << warningADIOS2NoGroupbasedEncoding << std::endl; printedWarningsAlready.noGroupBased = true; } fileData.m_IO.DefineAttribute( adios_defaults::str_groupBasedWarning, - std::string("Consider using file-based or variable-based " - "encoding instead in ADIOS2.")); + std::string(warningADIOS2NoGroupbasedEncoding)); } } } @@ -924,9 +931,22 @@ void ADIOS2IOHandlerImpl::openFile( writable->written = true; writable->abstractFilePosition = std::make_shared(); + auto how_to_open = [&]() { + switch (parameters.reopen) + { + case Parameter::Reopen::WasCreatedByUs: + return IfFileNotOpen::ReopenFileThatWeCreated; + case Parameter::Reopen::WasFoundOnDisk: + return IfFileNotOpen::ReopenFileFoundOnDisk; + case Parameter::Reopen::NoReopen: + return IfFileNotOpen::OpenImplicitly; + } + return IfFileNotOpen::ThrowError; // Unreachable + }(); + // enforce opening the file // lazy opening is deathly in parallel situations - auto &fileData = getFileData(file, IfFileNotOpen::OpenImplicitly); + auto &fileData = getFileData(file, how_to_open); *parameters.out_parsePreference = fileData.parsePreference; m_dirty.emplace(std::move(file)); } @@ -1053,25 +1073,7 @@ void ADIOS2IOHandlerImpl::writeAttribute( { return; } -#if openPMD_HAS_ADIOS_2_9 - switch (useGroupTable()) - { - case UseGroupTable::No: - if (parameters.changesOverSteps == - Parameter::ChangesOverSteps::Yes) - { - // cannot do this - return; - } - - break; - case UseGroupTable::Yes: { - break; - } - default: - throw std::runtime_error("Unreachable!"); - } -#else +#if !openPMD_HAS_ADIOS_2_9 if (parameters.changesOverSteps == Parameter::ChangesOverSteps::Yes) { @@ -1523,7 +1525,8 @@ void ADIOS2IOHandlerImpl::touch( m_dirty.emplace(std::move(file)); } -adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) +adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode( + std::string const &fullPath, adios_defs::OpenFileAs openFileAs) { if (m_config.json().contains("engine") && m_config["engine"].json().contains("access_mode")) @@ -1568,10 +1571,28 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) switch (m_handler->m_backendAccess) { case Access::CREATE: - return adios2::Mode::Write; + switch (openFileAs) + { + + case adios_defs::OpenFileAs::Create: + return adios2::Mode::Write; + case adios_defs::OpenFileAs::Open: + case adios_defs::OpenFileAs::ReopenFileThatWeCreated: + return adios2::Mode::Append; + } + break; #if openPMD_HAS_ADIOS_2_8 case Access::READ_LINEAR: - return adios2::Mode::Read; + switch (m_handler->m_encoding) + { + + case IterationEncoding::fileBased: + return adios2::Mode::ReadRandomAccess; + case IterationEncoding::groupBased: + case IterationEncoding::variableBased: + return adios2::Mode::Read; + } + break; case Access::READ_ONLY: return adios2::Mode::ReadRandomAccess; #else @@ -1583,11 +1604,36 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) if (auxiliary::directory_exists(fullPath) || auxiliary::file_exists(fullPath)) { + switch (m_handler->m_encoding) + { + + case IterationEncoding::fileBased: + switch (openFileAs) + { + + case adios_defs::OpenFileAs::Create: + return adios2::Mode::Write; + case adios_defs::OpenFileAs::Open: #if openPMD_HAS_ADIOS_2_8 - return adios2::Mode::ReadRandomAccess; + return adios2::Mode::ReadRandomAccess; #else - return adios2::Mode::Read; + return adios2::Mode::Read; #endif + case adios_defs::OpenFileAs::ReopenFileThatWeCreated: + /* In order to write new data to an Iteration that was + * created and closed previously, the only applicable access + * mode is Append mode, ideally in conjunction with + * `SetParameter("FlattenSteps", "ON")`. + * See test/Files_SerialIO/close_iteration_test.cpp. + */ + return adios2::Mode::Append; + } + break; + case IterationEncoding::groupBased: + case IterationEncoding::variableBased: + return adios2::Mode::Read; + } + break; } else { @@ -1686,21 +1732,35 @@ detail::ADIOS2File &ADIOS2IOHandlerImpl::getFileData( file.valid(), "[ADIOS2] Cannot retrieve file data for a file that has " "been overwritten or deleted.") + auto openFileAs = [&]() { + using OF = adios_defs::OpenFileAs; + switch (flag) + { + case IfFileNotOpen::ReopenFileThatWeCreated: + return OF::ReopenFileThatWeCreated; + case IfFileNotOpen::OpenImplicitly: + return OF::Open; + case IfFileNotOpen::CreateImplicitly: + return OF::Create; + case IfFileNotOpen::ThrowError: + break; + } + return OF{}; + }(); auto it = m_fileData.find(file); if (it == m_fileData.end()) { switch (flag) { - case IfFileNotOpen::OpenImplicitly: { - - auto res = m_fileData.emplace( - file, std::make_unique(*this, file)); - return *res.first->second; - } case IfFileNotOpen::ThrowError: throw std::runtime_error( "[ADIOS2] Requested file has not been opened yet: " + (file.fileState ? file.fileState->name : "Unknown file name")); + default: + auto res = m_fileData.emplace( + file, + std::make_unique(*this, file, openFileAs)); + return *res.first->second; } } else diff --git a/src/IO/AbstractIOHandler.cpp b/src/IO/AbstractIOHandler.cpp index 3d83978d1d..8f0bd4524e 100644 --- a/src/IO/AbstractIOHandler.cpp +++ b/src/IO/AbstractIOHandler.cpp @@ -90,6 +90,10 @@ void AbstractIOHandler::setIterationEncoding(IterationEncoding encoding) break; } } + else + { + m_backendAccess = m_frontendAccess; + } m_encoding = encoding; } diff --git a/src/IO/Access.cpp b/src/IO/Access.cpp new file mode 100644 index 0000000000..1112cee69c --- /dev/null +++ b/src/IO/Access.cpp @@ -0,0 +1,30 @@ +#include "openPMD/IO/Access.hpp" + +#include +#include + +namespace openPMD +{ +std::ostream &operator<<(std::ostream &o, Access const &a) +{ + switch (a) + { + case Access::READ_RANDOM_ACCESS: + o << std::string("READ_RANDOM_ACCESS"); + break; + case Access::READ_LINEAR: + o << std::string("READ_LINEAR"); + break; + case Access::READ_WRITE: + o << std::string("READ_WRITE"); + break; + case Access::CREATE: + o << std::string("CREATE"); + break; + case Access::APPEND: + o << std::string("APPEND"); + break; + } + return o; +} +} // namespace openPMD diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 28839fea3b..931a1f9e3d 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -21,16 +21,23 @@ #include "openPMD/Iteration.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" +#include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/Series.hpp" +#include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Writable.hpp" +#include #include #include +#include +#include +#include #include namespace openPMD @@ -94,21 +101,8 @@ Iteration &Iteration::close(bool _flush) case CloseStatus::ClosedInFrontend: it.m_closed = CloseStatus::ClosedInFrontend; break; - case CloseStatus::ClosedTemporarily: - // should we bother to reopen? - if (dirtyRecursive()) - { - // let's reopen - it.m_closed = CloseStatus::ClosedInFrontend; - } - else - { - // don't reopen - it.m_closed = CloseStatus::ClosedInBackend; - } - break; case CloseStatus::ParseAccessDeferred: - case CloseStatus::ClosedInBackend: + case CloseStatus::Closed: // just keep it like it is // (this means that closing an iteration that has not been parsed // yet keeps it re-openable) @@ -119,7 +113,7 @@ Iteration &Iteration::close(bool _flush) if (flag == StepStatus::DuringStep) { endStep(); - setStepStatus(StepStatus::NoStep); + setStepStatus(StepStatus::OutOfStep); } else { @@ -147,16 +141,28 @@ Iteration &Iteration::close(bool _flush) Iteration &Iteration::open() { + Series s = retrieveSeries(); auto &it = get(); + // figure out my iteration number + auto begin = s.indexOf(*this); + if (it.m_closed == internal::CloseStatus::ClosedInFrontend) + { + // Iteration is only logically closed, we can simply unmark it + it.m_closed = internal::CloseStatus::Open; + } + // Ensure that files are accessed. + // If the close status was Closed, this will open it. + s.openIteration(begin->first, *this); if (it.m_closed == CloseStatus::ParseAccessDeferred) { it.m_closed = CloseStatus::Open; runDeferredParseAccess(); } - Series s = retrieveSeries(); - // figure out my iteration number - auto begin = s.indexOf(*this); - s.openIteration(begin->first, *this); + if (getStepStatus() == StepStatus::OutOfStep) + { + beginStep(/* reread = */ false); + setStepStatus(StepStatus::DuringStep); + } IOHandler()->flush(internal::defaultFlushParams); return *this; } @@ -167,15 +173,28 @@ bool Iteration::closed() const { case CloseStatus::ParseAccessDeferred: case CloseStatus::Open: + return false; /* * Temporarily closing a file is something that the openPMD API * does for optimization purposes. * Logically to the user, it is still open. */ - case CloseStatus::ClosedTemporarily: + case CloseStatus::ClosedInFrontend: + case CloseStatus::Closed: + return true; + } + throw std::runtime_error("Unreachable!"); +} + +bool Iteration::parsed() const +{ + switch (get().m_closed) + { + case CloseStatus::ParseAccessDeferred: return false; + case CloseStatus::Open: case CloseStatus::ClosedInFrontend: - case CloseStatus::ClosedInBackend: + case CloseStatus::Closed: return true; } throw std::runtime_error("Unreachable!"); @@ -325,6 +344,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) m.second.flush(m.first, flushParams); for (auto &species : particles) species.second.flush(species.first, flushParams); + setDirty(false); } else { @@ -392,7 +412,10 @@ void Iteration::reread(std::string const &path) } void Iteration::readFileBased( - std::string const &filePath, std::string const &groupPath, bool doBeginStep) + IterationIndex_t idx, + std::string const &filePath, + std::string const &groupPath, + bool doBeginStep) { if (doBeginStep) { @@ -404,7 +427,15 @@ void Iteration::readFileBased( auto series = retrieveSeries(); series.readOneIterationFileBased(filePath); - get().m_overrideFilebasedFilename = filePath; + + auto &series_data = series.get(); + if (series_data.m_iterationFilenames.find(idx) == + series_data.m_iterationFilenames.end()) + { + throw error::Internal( + "[Iteration::readFileBased] Own filename should be placed into " + "buffer for later retrieval."); + } read_impl(groupPath); } @@ -693,10 +724,8 @@ auto Iteration::beginStep(bool reread) -> BeginStepStatus } auto Iteration::beginStep( - std::optional thisObject, - Series &series, - bool reread, - std::set const &ignoreIterations) -> BeginStepStatus + std::optional thisObject, Series &series, bool reread) + -> BeginStepStatus { BeginStepStatus res; using IE = IterationEncoding; @@ -708,7 +737,8 @@ auto Iteration::beginStep( case IE::fileBased: if (thisObject.has_value()) { - file = &static_cast(*thisObject).get(); + file = static_cast( + thisObject.value().m_attri.get()); } else { @@ -726,14 +756,13 @@ auto Iteration::beginStep( AdvanceStatus status; if (thisObject.has_value()) { + thisObject->setStepStatus(StepStatus::DuringStep); status = series.advance( - AdvanceMode::BEGINSTEP, - *file, - series.indexOf(*thisObject), - *thisObject); + AdvanceMode::BEGINSTEP, *file, series.indexOf(*thisObject)); } else { + series.get().m_stepStatus = StepStatus::DuringStep; status = series.advance(AdvanceMode::BEGINSTEP); } @@ -764,7 +793,7 @@ auto Iteration::beginStep( res.iterationsInOpenedStep = series.readGorVBased( /* do_always_throw_errors = */ true, /* init = */ false, - ignoreIterations); + /* read_only_this_single_iteration = */ std::nullopt); } catch (...) { @@ -775,6 +804,30 @@ auto Iteration::beginStep( series.iterations.setWritten( previous, Attributable::EnqueueAsynchronously::Yes); } + else if (thisObject.has_value()) + { + IterationIndex_t idx = series.indexOf(*thisObject)->first; + res.iterationsInOpenedStep = {idx}; + } + else + { + switch (status) + { + + case AdvanceStatus::OK: + throw error::Internal( + "Control flow error: Opening a new step requires reparsing."); + case AdvanceStatus::RANDOMACCESS: + std::transform( + series.iterations.begin(), + series.iterations.end(), + std::back_inserter(res.iterationsInOpenedStep), + [](auto const &pair) { return pair.first; }); + break; + case AdvanceStatus::OVER: + break; + } + } res.stepStatus = status; return res; @@ -790,7 +843,7 @@ void Iteration::endStep() switch (series.iterationEncoding()) { case IE::fileBased: - file = &Attributable::get(); + file = m_attri.get(); break; case IE::groupBased: case IE::variableBased: @@ -798,7 +851,7 @@ void Iteration::endStep() break; } // @todo filebased check - series.advance(AdvanceMode::ENDSTEP, *file, series.indexOf(*this), *this); + series.advance(AdvanceMode::ENDSTEP, *file, series.indexOf(*this)); series.get().m_currentlyActiveIterations.clear(); } @@ -860,8 +913,14 @@ void Iteration::runDeferredParseAccess() { if (deferred.fileBased) { + auto const &filename = + retrieveSeries().get().m_iterationFilenames.at( + deferred.iteration); readFileBased( - deferred.filename, deferred.path, deferred.beginStep); + deferred.iteration, + filename, + deferred.path, + deferred.beginStep); } else { diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp index fff398c8cd..a1cd619b9b 100644 --- a/src/ReadIterations.cpp +++ b/src/ReadIterations.cpp @@ -1,628 +1,34 @@ -/* Copyright 2021 Franz Poeschel - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ - #include "openPMD/ReadIterations.hpp" -#include "openPMD/Error.hpp" - -#include "openPMD/Series.hpp" - -#include -#include +#include "openPMD/snapshots/IteratorHelpers.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" namespace openPMD { +LegacyIteratorAdaptor::LegacyIteratorAdaptor(Snapshots::iterator iterator) + : m_iterator(std::move(iterator)) +{} -namespace -{ - bool reread(std::optional parsePreference) - { - if (parsePreference.has_value()) - { - using PP = Parameter::ParsePreference; - - switch (parsePreference.value()) - { - case PP::PerStep: - return true; - case PP::UpFront: - return false; - } - return false; - } - else - { - throw error::Internal( - "Group/Variable-based encoding: Parse preference must be set."); - } - } -} // namespace - -SeriesIterator::SeriesIterator() = default; - -void SeriesIterator::initSeriesInLinearReadMode() -{ - auto &data = get(); - auto &series = *data.series; - series.IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing; - try - { - switch (series.iterationEncoding()) - { - using IE = IterationEncoding; - case IE::fileBased: - series.readFileBased(); - break; - case IE::groupBased: - case IE::variableBased: { - Parameter fOpen; - fOpen.name = series.get().m_name; - series.IOHandler()->enqueue(IOTask(&series, fOpen)); - series.IOHandler()->flush(internal::defaultFlushParams); - using PP = Parameter::ParsePreference; - switch (*fOpen.out_parsePreference) - { - case PP::PerStep: - series.advance(AdvanceMode::BEGINSTEP); - series.readGorVBased( - /* do_always_throw_errors = */ false, /* init = */ true); - break; - case PP::UpFront: - series.readGorVBased( - /* do_always_throw_errors = */ false, /* init = */ true); - /* - * In linear read mode (where no parsing at all is done upon - * constructing the Series), it might turn out after parsing - * that what we expected to be a group-based Series was in fact - * a single file of a file-based Series. - * (E.g. when opening "data00000100.h5" directly instead of - * "data%T.h5") - * So we need to check the current value of - * `iterationEncoding()` once more. - */ - if (series.iterationEncoding() != IterationEncoding::fileBased) - { - series.advance(AdvanceMode::BEGINSTEP); - } - break; - } - data.parsePreference = *fOpen.out_parsePreference; - break; - } - } - } - catch (...) - { - series.IOHandler()->m_seriesStatus = internal::SeriesStatus::Default; - throw; - } - series.IOHandler()->m_seriesStatus = internal::SeriesStatus::Default; -} - -void SeriesIterator::close() -{ - *m_data = std::nullopt; // turn this into end iterator -} - -SeriesIterator::SeriesIterator( - Series const &series_in, - std::optional const &parsePreference) - : m_data{std::make_shared>(std::in_place)} -{ - auto &data = get(); - data.parsePreference = parsePreference; - /* - * Since the iterator is stored in - * internal::SeriesData::m_sharedStatefulIterator, - * we need to use a non-owning Series instance here for tie-breaking - * purposes. - * This is ok due to the usual C++ iterator invalidation workflows - * (deleting the original container invalidates the iterator). - */ - data.series = Series(); - data.series->setData(std::shared_ptr( - series_in.m_series.get(), [](auto const *) {})); - auto &series = data.series.value(); - if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR && - series.iterations.empty()) - { - initSeriesInLinearReadMode(); - } - - auto it = series.get().iterations.begin(); - if (it == series.get().iterations.end()) - { - this->close(); - return; - } - else if ( - it->second.get().m_closed == internal::CloseStatus::ClosedInBackend) - { - throw error::WrongAPIUsage( - "Trying to call Series::readIterations() on a (partially) read " - "Series."); - } - else - { - auto openIteration = [](Iteration &iteration) { - /* - * @todo - * Is that really clean? - * Use case: See Python ApiTest testListSeries: - * Call listSeries twice. - */ - if (iteration.get().m_closed != - internal::CloseStatus::ClosedInBackend) - { - iteration.open(); - } - }; - AdvanceStatus status{}; - switch (series.iterationEncoding()) - { - case IterationEncoding::fileBased: - /* - * The file needs to be accessed before beginning a step upon it. - * In file-based iteration layout it maybe is not accessed yet, - * so do that now. There is only one step per file, so beginning - * the step after parsing the file is ok. - */ - - openIteration(series.iterations.begin()->second); - status = it->second.beginStep(/* reread = */ true); - for (auto const &pair : series.iterations) - { - data.iterationsInCurrentStep.push_back(pair.first); - } - break; - case IterationEncoding::groupBased: - case IterationEncoding::variableBased: { - /* - * In group-based iteration layout, we have definitely already had - * access to the file until now. Better to begin a step right away, - * otherwise we might get another step's data. - */ - Iteration::BeginStepStatus::AvailableIterations_t - availableIterations; - std::tie(status, availableIterations) = it->second.beginStep( - /* reread = */ reread(data.parsePreference)); - /* - * In random-access mode, do not use the information read in the - * `snapshot` attribute, instead simply go through iterations - * one by one in ascending order (fallback implementation in the - * second if branch). - */ - if (availableIterations.has_value() && - status != AdvanceStatus::RANDOMACCESS) - { - data.iterationsInCurrentStep = availableIterations.value(); - if (!data.iterationsInCurrentStep.empty()) - { - openIteration(series.iterations.at( - data.iterationsInCurrentStep.at(0))); - } - } - else if (!series.iterations.empty()) - { - /* - * Fallback implementation: Assume that each step corresponds - * with an iteration in ascending order. - */ - data.iterationsInCurrentStep = { - series.iterations.begin()->first}; - openIteration(series.iterations.begin()->second); - } - else - { - // this is a no-op, but let's keep it explicit - data.iterationsInCurrentStep = {}; - } - - break; - } - } - - if (status == AdvanceStatus::OVER) - { - this->close(); - return; - } - if (!setCurrentIteration()) - { - this->close(); - return; - } - it->second.setStepStatus(StepStatus::DuringStep); - } -} - -std::optional SeriesIterator::nextIterationInStep() -{ - auto &data = get(); - using ret_t = std::optional; - - if (data.iterationsInCurrentStep.empty()) - { - return ret_t{}; - } - data.iterationsInCurrentStep.pop_front(); - if (data.iterationsInCurrentStep.empty()) - { - return ret_t{}; - } - auto oldIterationIndex = data.currentIteration; - data.currentIteration = *data.iterationsInCurrentStep.begin(); - auto &series = data.series.value(); - - switch (series.iterationEncoding()) - { - case IterationEncoding::groupBased: - case IterationEncoding::variableBased: { - auto begin = series.iterations.find(oldIterationIndex); - auto end = begin; - ++end; - series.flush_impl( - begin, - end, - {FlushLevel::UserFlush}, - /* flushIOHandler = */ true); - - try - { - series.iterations[data.currentIteration].open(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read iteration '" << data.currentIteration - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - return nextIterationInStep(); - } - - return {this}; - } - case IterationEncoding::fileBased: - try - { - /* - * Errors in here might appear due to deferred iteration parsing. - */ - series.iterations[data.currentIteration].open(); - /* - * Errors in here might appear due to reparsing after opening a - * new step. - */ - series.iterations[data.currentIteration].beginStep( - /* reread = */ true); - } - catch (error::ReadError const &err) - { - std::cerr << "[SeriesIterator] Cannot read iteration due to error " - "below, will skip it.\n" - << err.what() << std::endl; - return nextIterationInStep(); - } - - return {this}; - } - throw std::runtime_error("Unreachable!"); -} - -std::optional SeriesIterator::nextStep(size_t recursion_depth) -{ - auto &data = get(); - // since we are in group-based iteration layout, it does not - // matter which iteration we begin a step upon - AdvanceStatus status{}; - Iteration::BeginStepStatus::AvailableIterations_t availableIterations; - try - { - std::tie(status, availableIterations) = Iteration::beginStep( - {}, - *data.series, - /* reread = */ reread(data.parsePreference), - data.ignoreIterations); - } - catch (error::ReadError const &err) - { - std::cerr << "[SeriesIterator] Cannot read iteration due to error " - "below, will skip it.\n" - << err.what() << std::endl; - data.series->advance(AdvanceMode::ENDSTEP); - return nextStep(recursion_depth + 1); - } - - if (availableIterations.has_value() && - status != AdvanceStatus::RANDOMACCESS) - { - data.iterationsInCurrentStep = availableIterations.value(); - } - else - { - /* - * Fallback implementation: Assume that each step corresponds - * with an iteration in ascending order. - */ - auto &series = data.series.value(); - auto it = series.iterations.find(data.currentIteration); - auto itEnd = series.iterations.end(); - if (it == itEnd) - { - if (status == AdvanceStatus::RANDOMACCESS || - status == AdvanceStatus::OVER) - { - this->close(); - return {this}; - } - else - { - /* - * Stream still going but there was no iteration found in the - * current IO step? - * Might be a duplicate iteration resulting from appending, - * will skip such iterations and hope to find something in a - * later IO step. No need to finish right now. - */ - data.iterationsInCurrentStep = {}; - } - } - else - { - for (size_t i = 0; i < recursion_depth && it != itEnd; ++i) - { - ++it; - } - - if (it == itEnd) - { - if (status == AdvanceStatus::RANDOMACCESS || - status == AdvanceStatus::OVER) - { - this->close(); - return {this}; - } - else - { - /* - * Stream still going but there was no iteration found in - * the current IO step? Might be a duplicate iteration - * resulting from appending, will skip such iterations and - * hope to find something in a later IO step. No need to - * finish right now. - */ - data.iterationsInCurrentStep = {}; - } - } - else - { - data.iterationsInCurrentStep = {it->first}; - } - } - } - - if (status == AdvanceStatus::OVER) - { - this->close(); - return {this}; - } - - return {this}; -} - -std::optional SeriesIterator::loopBody() -{ - auto &data = get(); - Series &series = data.series.value(); - auto &iterations = series.iterations; - - /* - * Might not be present because parsing might have failed in previous step - */ - if (iterations.contains(data.currentIteration)) - { - auto ¤tIteration = iterations[data.currentIteration]; - if (!currentIteration.closed()) - { - currentIteration.close(); - } - } - - auto guardReturn = - [&series, &iterations]( - auto const &option) -> std::optional { - if (!option.has_value() || *option.value() == end()) - { - return option; - } - auto currentIterationIndex = option.value()->peekCurrentIteration(); - if (!currentIterationIndex.has_value()) - { - series.advance(AdvanceMode::ENDSTEP); - return std::nullopt; - } - // If we had the iteration already, then it's either not there at all - // (because old iterations are deleted in linear access mode), - // or it's still there but closed in random-access mode - auto index = currentIterationIndex.value(); - - if (iterations.contains(index)) - { - auto iteration = iterations.at(index); - if (iteration.get().m_closed != - internal::CloseStatus::ClosedInBackend) - { - try - { - iterations.at(index).open(); - option.value()->setCurrentIteration(); - return option; - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read iteration '" - << currentIterationIndex.value() - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - option.value()->deactivateDeadIteration( - currentIterationIndex.value()); - return std::nullopt; - } - } - else - { - // we had this iteration already, skip it - iteration.endStep(); - return std::nullopt; // empty, go into next iteration - } - } - else - { - // we had this iteration already, skip it - series.advance(AdvanceMode::ENDSTEP); - return std::nullopt; - } - }; - - { - auto optionallyAStep = nextIterationInStep(); - if (optionallyAStep.has_value()) - { - return guardReturn(optionallyAStep); - } - } - - // The currently active iterations have been exhausted. - // Now see if there are further iterations to be found. - - if (series.iterationEncoding() == IterationEncoding::fileBased) - { - // this one is handled above, stream is over once it proceeds to here - this->close(); - return {this}; - } - - auto option = nextStep(/*recursion_depth = */ 1); - return guardReturn(option); -} - -void SeriesIterator::deactivateDeadIteration(iteration_index_t index) -{ - auto &data = get(); - switch (data.series->iterationEncoding()) - { - case IterationEncoding::fileBased: { - Parameter param; - data.series->IOHandler()->enqueue( - IOTask(&data.series->iterations[index], std::move(param))); - data.series->IOHandler()->flush({FlushLevel::UserFlush}); - } - break; - case IterationEncoding::variableBased: - case IterationEncoding::groupBased: { - Parameter param; - param.mode = AdvanceMode::ENDSTEP; - data.series->IOHandler()->enqueue( - IOTask(&data.series->iterations[index], std::move(param))); - data.series->IOHandler()->flush({FlushLevel::UserFlush}); - } - break; - } - data.series->iterations.container().erase(index); -} - -SeriesIterator &SeriesIterator::operator++() -{ - auto &data = get(); - if (!data.series.has_value()) - { - this->close(); - return *this; - } - auto oldIterationIndex = data.currentIteration; - std::optional res; - /* - * loopBody() might return an empty option to indicate a skipped iteration. - * Loop until it returns something real for us. - * Note that this is not an infinite loop: - * Upon end of the Series, loopBody() does not return an empty option, - * but the end iterator. - */ - do - { - res = loopBody(); - } while (!res.has_value()); - - auto resvalue = res.value(); - if (*resvalue != end()) - { - auto &series = data.series.value(); - auto index = data.currentIteration; - auto &iteration = series.iterations[index]; - iteration.setStepStatus(StepStatus::DuringStep); - - if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR) - { - /* - * Linear read mode: Any data outside the current iteration is - * inaccessible. Delete the iteration. This has two effects: - * - * 1) Avoid confusion. - * 2) Avoid memory buildup in long-running workflows with many - * iterations. - * - * @todo Also delete data in the backends upon doing this. - */ - auto &container = series.iterations.container(); - container.erase(oldIterationIndex); - data.ignoreIterations.emplace(oldIterationIndex); - } - } - return *resvalue; -} - -IndexedIteration SeriesIterator::operator*() +auto LegacyIteratorAdaptor::operator*() const -> value_type { - auto &data = get(); - return IndexedIteration( - data.series.value().iterations[data.currentIteration], - data.currentIteration); + return m_iterator.operator*(); } -bool SeriesIterator::operator==(SeriesIterator const &other) const +auto LegacyIteratorAdaptor::operator++() -> LegacyIteratorAdaptor & { - return - // either both iterators are filled - (this->m_data->has_value() && other.m_data->has_value() && - (this->get().currentIteration == other.get().currentIteration)) || - // or both are empty - (!this->m_data->has_value() && !other.m_data->has_value()); + ++m_iterator; + return *this; } -bool SeriesIterator::operator!=(SeriesIterator const &other) const +auto LegacyIteratorAdaptor::operator==(LegacyIteratorAdaptor const &other) const + -> bool { - return !operator==(other); + return m_iterator == other.m_iterator; } -SeriesIterator SeriesIterator::end() +auto LegacyIteratorAdaptor::operator!=(LegacyIteratorAdaptor const &other) const + -> bool { - return SeriesIterator{}; + return m_iterator != other.m_iterator; } ReadIterations::ReadIterations( @@ -635,24 +41,24 @@ ReadIterations::ReadIterations( if (access == Access::READ_LINEAR && !data.m_sharedStatefulIterator) { // Open the iterator now already, so that metadata may already be read - data.m_sharedStatefulIterator = - std::make_unique(m_series, m_parsePreference); + data.m_sharedStatefulIterator = std::make_unique( + StatefulIterator::tag_read, m_series, m_parsePreference); } } -ReadIterations::iterator_t ReadIterations::begin() +auto ReadIterations::begin() -> iterator_t { auto &series = m_series.get(); if (!series.m_sharedStatefulIterator) { - series.m_sharedStatefulIterator = - std::make_unique(m_series, m_parsePreference); + series.m_sharedStatefulIterator = std::make_unique( + StatefulIterator::tag_read, m_series, m_parsePreference); } - return *series.m_sharedStatefulIterator; + return stateful_to_opaque(*series.m_sharedStatefulIterator); } -ReadIterations::iterator_t ReadIterations::end() +auto ReadIterations::end() -> iterator_t { - return SeriesIterator::end(); + return stateful_to_opaque(StatefulIterator::end()); } } // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index 2d10575c17..cfd92f84e7 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -28,8 +28,8 @@ #include "openPMD/IO/DummyIOHandler.hpp" #include "openPMD/IO/Format.hpp" #include "openPMD/IO/IOTask.hpp" +#include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" -#include "openPMD/ReadIterations.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Date.hpp" #include "openPMD/auxiliary/Filesystem.hpp" @@ -38,6 +38,11 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/snapshots/ContainerImpls.hpp" +#include "openPMD/snapshots/IteratorTraits.hpp" +#include "openPMD/snapshots/RandomAccessIterator.hpp" +#include "openPMD/snapshots/Snapshots.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/version.hpp" #include @@ -53,6 +58,7 @@ #include #include #include +#include namespace openPMD { @@ -1072,7 +1078,7 @@ void Series::initSeries( std::unique_ptr input) { auto &series = get(); - auto &writable = series.m_writable; + auto &writable = series->m_writable; /* * In Access modes READ_LINEAR and APPEND, the Series constructor might have @@ -1147,10 +1153,13 @@ Given file pattern: ')END" try { if (input->iterationEncoding == IterationEncoding::fileBased) - readFileBased(); + readFileBased( + /* read_only_this_single_iteration = */ std::nullopt); else readGorVBased( - /* do_always_throw_errors = */ false, /* init = */ true); + /* do_always_throw_errors = */ false, + /* init = */ true, + /* read_only_this_single_iteration = */ std::nullopt); if (series.iterations.empty()) { @@ -1336,8 +1345,7 @@ void Series::flushFileBased( { Parameter fClose; IOHandler()->enqueue(IOTask(&it->second, std::move(fClose))); - it->second.get().m_closed = - internal::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::Closed; } } @@ -1396,8 +1404,7 @@ void Series::flushFileBased( { Parameter fClose; IOHandler()->enqueue(IOTask(&it->second, std::move(fClose))); - it->second.get().m_closed = - internal::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::Closed; } /* reset the dirty bit for every iteration (i.e. file) * otherwise only the first iteration will have updates attributes @@ -1448,8 +1455,7 @@ void Series::flushGorVBased( internal::CloseStatus::ClosedInFrontend) { // the iteration has no dedicated file in group-based mode - it->second.get().m_closed = - internal::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::Closed; } } @@ -1527,8 +1533,7 @@ void Series::flushGorVBased( internal::CloseStatus::ClosedInFrontend) { // the iteration has no dedicated file in group-based mode - it->second.get().m_closed = - internal::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::Closed; } } @@ -1562,7 +1567,8 @@ void Series::flushParticlesPath() IOHandler()->enqueue(IOTask(this, aWrite)); } -void Series::readFileBased() +void Series::readFileBased( + std::optional read_only_this_single_iteration) { auto &series = get(); Parameter fOpen; @@ -1592,14 +1598,18 @@ void Series::readFileBased() isPartOfSeries, IOHandler()->directory, // foreach found file with `filename` and `index`: - [&series](std::string const &filename, Match const &match) { + [&series, &read_only_this_single_iteration]( + std::string const &filename, Match const &match) { auto index = match.iteration; + if (read_only_this_single_iteration.has_value() && + *read_only_this_single_iteration != index) + { + return; + } Iteration &i = series.iterations[index]; - i.deferParseAccess( - {std::to_string(index), - index, - true, - cleanFilename(filename, series.m_filenameExtension).body}); + series.m_iterationFilenames[index] = + cleanFilename(filename, series.m_filenameExtension).body; + i.deferParseAccess({std::to_string(index), index, true}); }); if (series.iterations.empty()) @@ -1622,7 +1632,8 @@ void Series::readFileBased() * Return true if parsing was successful */ auto readIterationEagerly = - [](Iteration &iteration) -> std::optional { + [&read_only_this_single_iteration]( + Iteration &iteration) -> std::optional { try { iteration.runDeferredParseAccess(); @@ -1631,10 +1642,16 @@ void Series::readFileBased() { return err; } - Parameter fClose; - iteration.IOHandler()->enqueue(IOTask(&iteration, fClose)); - iteration.IOHandler()->flush(internal::defaultFlushParams); - iteration.get().m_closed = internal::CloseStatus::ClosedTemporarily; + // If one specific iteration is requested, keep it open afterward. + if (!read_only_this_single_iteration.has_value()) + { + Parameter fClose; + iteration.IOHandler()->enqueue(IOTask(&iteration, fClose)); + iteration.IOHandler()->flush(internal::defaultFlushParams); + auto &it_data = iteration.get(); + it_data.m_closed = internal::CloseStatus::Closed; + it_data.allow_reopening_implicitly = true; + } return {}; }; std::vector unparseableIterations; @@ -1650,6 +1667,11 @@ void Series::readFileBased() std::optional forwardFirstError; for (auto &pair : series.iterations) { + if (read_only_this_single_iteration.has_value() && + *read_only_this_single_iteration != pair.first) + { + continue; + } if (auto error = readIterationEagerly(pair.second); error) { std::cerr << "Cannot read iteration '" << pair.first @@ -1699,6 +1721,11 @@ void Series::readFileBased() std::optional forwardFirstError; for (auto &iteration : series.iterations) { + if (read_only_this_single_iteration.has_value() && + *read_only_this_single_iteration != iteration.first) + { + continue; + } if (auto error = readIterationEagerly(iteration.second); error) { std::cerr << "Cannot read iteration '" << iteration.first @@ -1761,6 +1788,12 @@ void Series::readOneIterationFileBased(std::string const &filePath) { auto &series = get(); + IOHandler()->m_encoding = IterationEncoding::fileBased; + // In this case, READ_LINEAR is implemented exclusively in the frontend + if (IOHandler()->m_backendAccess == Access::READ_LINEAR) + { + IOHandler()->m_backendAccess = Access::READ_ONLY; + } Parameter fOpen; Parameter aRead; @@ -1775,14 +1808,15 @@ void Series::readOneIterationFileBased(std::string const &filePath) aRead.name = "iterationEncoding"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); + IterationEncoding encoding_out; if (*aRead.dtype == DT::STRING) { std::string encoding = Attribute(*aRead.resource).get(); if (encoding == "fileBased") - series.m_iterationEncoding = IterationEncoding::fileBased; + encoding_out = IterationEncoding::fileBased; else if (encoding == "groupBased") { - series.m_iterationEncoding = IterationEncoding::fileBased; + encoding_out = IterationEncoding::fileBased; std::cerr << "Series constructor called with iteration regex '%T' " "suggests loading a time series with fileBased iteration " @@ -1812,7 +1846,10 @@ void Series::readOneIterationFileBased(std::string const &filePath) error::Reason::UnexpectedContent, {}, "Unknown iterationEncoding: " + encoding); - setAttribute("iterationEncoding", encoding); + auto old_written = written(); + setWritten(false, Attributable::EnqueueAsynchronously::No); + setIterationEncoding(encoding_out); + setWritten(old_written, Attributable::EnqueueAsynchronously::Yes); } else throw std::runtime_error( @@ -1864,9 +1901,8 @@ namespace * otherwise. Use only where an empty subtract vector is the * common case. */ - template - void - vectorDifference(std::vector &baseVector, std::vector const &subtract) + template + void vectorDifference(V1 &baseVector, V2 const &subtract) { for (auto const &elem : subtract) { @@ -1885,8 +1921,8 @@ namespace auto Series::readGorVBased( bool do_always_throw_errors, bool do_init, - std::set const &ignoreIterations) - -> std::optional> + std::optional read_only_this_single_iteration) + -> std::vector { auto &series = get(); Parameter fOpen; @@ -2025,7 +2061,6 @@ creating new iterations. [&series, &pOpen, this]( IterationIndex_t index, std::string const &path, - bool guardAgainstRereading, bool beginStep) -> std::optional { if (series.iterations.contains(index)) { @@ -2033,7 +2068,7 @@ creating new iterations. auto &i = series.iterations.at(index); // i.written(): the iteration has already been parsed // reparsing is not needed - if (guardAgainstRereading && i.written()) + if (i.written()) { return {}; } @@ -2051,7 +2086,7 @@ creating new iterations. { // parse for the first time, resp. delay the parsing process Iteration &i = series.iterations[index]; - i.deferParseAccess({path, index, false, "", beginStep}); + i.deferParseAccess({path, index, false, beginStep}); if (!series.m_parseLazily) { try @@ -2085,19 +2120,20 @@ creating new iterations. * Sic! This happens when a file-based Series is opened in group-based mode. */ case IterationEncoding::fileBased: { - std::vector unreadableIterations; + std::vector unreadableIterations; + std::vector readableIterations; + readableIterations.reserve(pList.paths->size()); for (auto const &it : *pList.paths) { IterationIndex_t index = std::stoull(it); - if (ignoreIterations.find(index) != ignoreIterations.end()) + if (read_only_this_single_iteration.has_value() && + index != *read_only_this_single_iteration) { continue; } if (auto err = internal::withRWAccess( IOHandler()->m_seriesStatus, - [&]() { - return readSingleIteration(index, it, true, false); - }); + [&]() { return readSingleIteration(index, it, false); }); err) { std::cerr << "Cannot read iteration " << index @@ -2109,35 +2145,46 @@ creating new iterations. } unreadableIterations.push_back(index); } + else + { + readableIterations.push_back(index); + } } if (currentSteps.has_value()) { auto &vec = currentSteps.value(); vectorDifference(vec, unreadableIterations); - return std::deque{vec.begin(), vec.end()}; + return vec; } else { - return std::optional>(); + return readableIterations; } } case IterationEncoding::variableBased: { - std::deque res{}; - if (currentSteps.has_value() && !currentSteps.value().empty()) + if (!currentSteps.has_value() || currentSteps.value().empty()) { - for (auto index : currentSteps.value()) - { - if (ignoreIterations.find(index) == ignoreIterations.end()) - { - res.push_back(index); - } - } + currentSteps = std::vector{ + read_only_this_single_iteration.has_value() + ? *read_only_this_single_iteration + : 0}; } - else + else if (read_only_this_single_iteration.has_value()) { - res = {0}; + if (std::find( + currentSteps->begin(), + currentSteps->end(), + *read_only_this_single_iteration) != currentSteps->end()) + { + *currentSteps = {*read_only_this_single_iteration}; + } + else + { + currentSteps->clear(); + } } - for (auto it : res) + + for (auto it : *currentSteps) { /* * Variable-based iteration encoding relies on steps, so parsing @@ -2146,7 +2193,7 @@ creating new iterations. if (auto err = internal::withRWAccess( IOHandler()->m_seriesStatus, [&readSingleIteration, it]() { - return readSingleIteration(it, "", false, true); + return readSingleIteration(it, "", true); }); err) { @@ -2159,7 +2206,7 @@ creating new iterations. throw *err; } } - return res; + return *currentSteps; } } throw std::runtime_error("Unreachable!"); @@ -2263,6 +2310,12 @@ void Series::readBase() "string, found " + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); } + else + { + // Make sure that the meshesPath does not leak from one iteration into + // the other in file-based iteration encoding + Attributable::get().m_attributes.erase("meshesPath"); + } if (std::count( aList.attributes->begin(), @@ -2295,6 +2348,11 @@ void Series::readBase() "string, found " + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); } + { + // Make sure that the particlesPath does not leak from one iteration + // into the other in file-based iteration encoding + Attributable::get().m_attributes.erase("particlesPath"); + } } std::string Series::iterationFilename(IterationIndex_t i) @@ -2308,11 +2366,10 @@ std::string Series::iterationFilename(IterationIndex_t i) { return series.m_overrideFilebasedFilename.value(); } - else if (auto iteration = iterations.find(i); // - iteration != iterations.end() && - iteration->second.get().m_overrideFilebasedFilename.has_value()) + else if (auto iteration = series.m_iterationFilenames.find(i); // + iteration != series.m_iterationFilenames.end()) { - return iteration->second.get().m_overrideFilebasedFilename.value(); + return iteration->second; } else { @@ -2347,13 +2404,14 @@ Series::iterations_iterator Series::indexOf(Iteration const &iteration) AdvanceStatus Series::advance( AdvanceMode mode, internal::AttributableData &file, - iterations_iterator begin, - Iteration &iteration) + iterations_iterator begin) { internal::FlushParams const flushParams = {FlushLevel::UserFlush}; auto &series = get(); auto end = begin; ++end; + + auto &iteration = begin->second; /* * We call flush_impl() with flushIOHandler = false, meaning that tasks are * not yet propagated to the backend. @@ -2369,6 +2427,9 @@ AdvanceStatus Series::advance( itData.m_closed = internal::CloseStatus::Open; } + bool old_dirty = iteration.dirty(); + iteration.setDirty(true); // force flush() to open this + switch (mode) { case AdvanceMode::ENDSTEP: @@ -2385,6 +2446,7 @@ AdvanceStatus Series::advance( end, {FlushLevel::CreateOrOpenFiles}, /* flushIOHandler = */ false); + iteration.setDirty(old_dirty); break; } @@ -2392,10 +2454,10 @@ AdvanceStatus Series::advance( { // Series::flush() would normally turn a `ClosedInFrontend` into // a `ClosedInBackend`. Do that manually. - itData.m_closed = internal::CloseStatus::ClosedInBackend; + itData.m_closed = internal::CloseStatus::Closed; } else if ( - oldCloseStatus == internal::CloseStatus::ClosedInBackend && + oldCloseStatus == internal::CloseStatus::Closed && series.m_iterationEncoding == IterationEncoding::fileBased) { /* @@ -2403,7 +2465,7 @@ AdvanceStatus Series::advance( * opening an iteration's file by beginning a step on it. * So, return now. */ - iteration.get().m_closed = internal::CloseStatus::ClosedInBackend; + iteration.get().m_closed = internal::CloseStatus::Closed; return AdvanceStatus::OK; } @@ -2413,7 +2475,7 @@ AdvanceStatus Series::advance( } Parameter param; - if (itData.m_closed == internal::CloseStatus::ClosedTemporarily && + if (itData.m_closed == internal::CloseStatus::Closed && series.m_iterationEncoding == IterationEncoding::fileBased) { /* @@ -2433,7 +2495,7 @@ AdvanceStatus Series::advance( // If the backend does not support steps, we cannot continue here param.isThisStepMandatory = true; } - IOTask task(&file.m_writable, param); + IOTask task(&file->m_writable, param); IOHandler()->enqueue(task); } @@ -2444,12 +2506,9 @@ AdvanceStatus Series::advance( switch (series.m_iterationEncoding) { case IE::fileBased: { - if (itData.m_closed != internal::CloseStatus::ClosedTemporarily) - { - Parameter fClose; - IOHandler()->enqueue(IOTask(&iteration, std::move(fClose))); - } - itData.m_closed = internal::CloseStatus::ClosedInBackend; + Parameter fClose; + IOHandler()->enqueue(IOTask(&iteration, std::move(fClose))); + itData.m_closed = internal::CloseStatus::Closed; break; } case IE::groupBased: { @@ -2459,7 +2518,7 @@ AdvanceStatus Series::advance( // In group-based iteration layout, files are // not closed on a per-iteration basis // We will treat it as such nonetheless - itData.m_closed = internal::CloseStatus::ClosedInBackend; + itData.m_closed = internal::CloseStatus::Closed; break; } case IE::variableBased: // no action necessary @@ -2530,7 +2589,7 @@ AdvanceStatus Series::advance(AdvanceMode mode) // If the backend does not support steps, we cannot continue here param.isThisStepMandatory = true; } - IOTask task(&series.m_writable, param); + IOTask task(&series->m_writable, param); IOHandler()->enqueue(task); // We cannot call Series::flush now, since the IO handler is still filled @@ -2571,19 +2630,20 @@ void Series::flushStep(bool doFlush) series.m_wroteAtLeastOneIOStep = true; } -auto Series::openIterationIfDirty(IterationIndex_t index, Iteration iteration) +auto Series::openIterationIfDirty(IterationIndex_t index, Iteration &iteration) -> IterationOpened { + auto &data = iteration.get(); /* * Check side conditions on accessing iterations, and if they are fulfilled, * forward function params to openIteration(). */ - if (iteration.get().m_closed == internal::CloseStatus::ParseAccessDeferred) + if (data.m_closed == internal::CloseStatus::ParseAccessDeferred) { return IterationOpened::RemainsClosed; } bool const dirtyRecursive = iteration.dirtyRecursive(); - if (iteration.get().m_closed == internal::CloseStatus::ClosedInBackend) + if (data.m_closed == internal::CloseStatus::Closed) { // file corresponding with the iteration has previously been // closed and fully flushed @@ -2594,13 +2654,21 @@ auto Series::openIterationIfDirty(IterationIndex_t index, Iteration iteration) "[Series] Closed iteration has not been written. This " "is an internal error."); } - if (dirtyRecursive) + else if (!dirtyRecursive) { - throw std::runtime_error( - "[Series] Detected illegal access to iteration that " - "has been closed previously."); + return IterationOpened::RemainsClosed; + } + else if (!data.allow_reopening_implicitly) + { + throw error::WrongAPIUsage( + "[Series] Closed iteration (idx=" + std::to_string(index) + + ") must be open()ed explicitly before interacting with it " + "again."); + } + else + { + data.allow_reopening_implicitly = false; // only allow this once } - return IterationOpened::RemainsClosed; } switch (iterationEncoding()) @@ -2633,21 +2701,35 @@ auto Series::openIterationIfDirty(IterationIndex_t index, Iteration iteration) return IterationOpened::RemainsClosed; } -void Series::openIteration(IterationIndex_t index, Iteration iteration) +void Series::openIteration(IterationIndex_t index, Iteration &iteration) { auto oldStatus = iteration.get().m_closed; + using CL = internal::CloseStatus; + /* + * Closed and ClosedInFrontend need to be treated different here. + * Closed means that the Iteration is actually closed, but we need it again, + * so it should be opened again. + * ClosedInFrontend means that the Iteration is about to be closed, but + * still open. Nothing needs to be done, the enqueued operations can be + * performed, and the Iteration will be closed afterwards. + */ switch (oldStatus) { - using CL = internal::CloseStatus; - case CL::ClosedInBackend: - throw std::runtime_error( - "[Series] Detected illegal access to iteration that " - "has been closed previously."); - case CL::ParseAccessDeferred: + case CL::Closed: + if (access::writeOnly(IOHandler()->m_frontendAccess)) + { + std::cerr << &R"( +[Series::openIteration] + Warning: Reopening closed Iterations in write modes is currently experimental. + Note that an ADIOS2 step/file cannot be modified once closed, just appended + to with a new step. Support for this is not yet feature-complete (pre-alpha). +)"[1]; + } + [[fallthrough]]; case CL::Open: - case CL::ClosedTemporarily: iteration.get().m_closed = CL::Open; break; + case CL::ParseAccessDeferred: case CL::ClosedInFrontend: // just keep it like it is break; @@ -2683,6 +2765,26 @@ void Series::openIteration(IterationIndex_t index, Iteration iteration) // open the iteration's file again Parameter fOpen; fOpen.name = iterationFilename(index); + using R = Parameter::Reopen; + if (oldStatus != CL::Closed) + { + fOpen.reopen = R::NoReopen; + } + else if ( + // The filename only gets emplaced in there if we found it on the + // file system, otherwise it's generated by iterationFilename(). + // This helps us distinguish which iterations were created by us and + // which ones existed already. This is important for ADIOS2 which + // can open an iteration for Appending XOR for reading. + series.m_iterationFilenames.find(index) == + series.m_iterationFilenames.end()) + { + fOpen.reopen = R::WasCreatedByUs; + } + else + { + fOpen.reopen = R::WasFoundOnDisk; + } IOHandler()->enqueue(IOTask(this, fOpen)); /* open base path */ @@ -2861,9 +2963,9 @@ namespace internal void SeriesData::close() { // WriteIterations gets the first shot at flushing - if (this->m_writeIterations.has_value()) + if (this->m_sharedStatefulIterator) { - this->m_writeIterations.value().close(); + this->m_sharedStatefulIterator->close(); } /* * Scenario: A user calls `Series::flush()` but does not check for @@ -2894,9 +2996,9 @@ namespace internal // This releases the openPMD hierarchy iterations.container().clear(); // Release the IO Handler - if (m_writable.IOHandler) + if (operator*().m_writable.IOHandler) { - *m_writable.IOHandler = std::nullopt; + *operator*().m_writable.IOHandler = std::nullopt; } } } // namespace internal @@ -2936,10 +3038,165 @@ ReadIterations Series::readIterations() { // Use private constructor instead of copy constructor to avoid // object slicing - Series res; - res.setData(std::dynamic_pointer_cast(this->m_attri)); + auto series = m_attri->asInternalCopyOf(); return ReadIterations{ - std::move(res), IOHandler()->m_frontendAccess, get().m_parsePreference}; + std::move(series), + IOHandler()->m_frontendAccess, + get().m_parsePreference}; +} + +namespace +{ + auto make_writing_stateful_iterator( + Series const &copied_series, internal::SeriesData &series) + -> std::function + { + if (!series.m_sharedStatefulIterator) + { + series.m_sharedStatefulIterator = + std::make_unique( + StatefulIterator::tag_write, copied_series); + } + return [ptr = series.m_sharedStatefulIterator.get()]() { return ptr; }; + } + auto make_reading_stateful_iterator( + Series copied_series, internal::SeriesData &series) + -> std::function + { + return [s = std::move(copied_series), &series]() mutable { + if (!series.m_sharedStatefulIterator) + { + auto parse_preference = series.m_parsePreference; + series.m_sharedStatefulIterator = + std::make_unique( + StatefulIterator::tag_read, + std::move(s), + parse_preference); + } + return series.m_sharedStatefulIterator.get(); + }; + } +} // namespace + +Snapshots +Series::snapshots(std::optional const snapshot_workflow) +{ + auto &series = get(); + if (series.m_deferred_initialization.has_value()) + { + runDeferredInitialization(); + } + auto access = IOHandler()->m_frontendAccess; + auto guard_wrong_access_specification = + [&](SnapshotWorkflow required_access) { + if (!snapshot_workflow.has_value()) + { + return required_access; + } + if (required_access != *snapshot_workflow) + { + std::stringstream error; + error << "[Series::snapshots()] Specified " + << (*snapshot_workflow == SnapshotWorkflow::Synchronous + ? "linear" + : "random-access") + << " iteration in method parameter " + "`snapshot_workflow`, but access type " + << access << " requires " + << (required_access == SnapshotWorkflow::Synchronous + ? "linear" + : "random-access") + << " iteration. Please remove the parameter, there is no " + "need to specify it under " + << access << " mode." << std::endl; + throw error::WrongAPIUsage(error.str()); + } + else + { + std::cerr + << "[Series::snapshots()] No need to explicitly specify " + "synchronous or non-synchronous access via method " + "parameter `snapshot_workflow` in mode '" + << access << ". Will ignore." << std::endl; + } + return required_access; + }; + SnapshotWorkflow usedSnapshotWorkflow{}; + { + switch (access) + { + case Access::READ_LINEAR: + usedSnapshotWorkflow = + guard_wrong_access_specification(SnapshotWorkflow::Synchronous); + break; + case Access::READ_ONLY: + usedSnapshotWorkflow = guard_wrong_access_specification( + SnapshotWorkflow::RandomAccess); + + // Some error checks + if (series.m_parsePreference.has_value()) + { + switch (series.m_parsePreference.value()) + { + case internal::ParsePreference::UpFront: + break; + case internal::ParsePreference::PerStep: + throw error::ReadError( + error::AffectedObject::File, + error::Reason::UnexpectedContent, + std::nullopt, + "[Series::snapshots()] Series requires collective " + "processing with READ_LINEAR access mode."); + } + } + else if (iterationEncoding() != IterationEncoding::fileBased) + { + throw error::Internal( + "READ_ONLY mode and non-fileBased iteration encoding, but " + "the backend did not set a parse preference."); + } + break; + case Access::READ_WRITE: + // Our Read-Write workflows are entirely random-access based (so + // far). + // (Might be possible to allow stateful access actually, but there's + // no real use, so keep it simple.) + usedSnapshotWorkflow = guard_wrong_access_specification( + SnapshotWorkflow::RandomAccess); + break; + + case Access::CREATE: + case Access::APPEND: + // Users can select. + usedSnapshotWorkflow = snapshot_workflow.value_or( + /* random-access logic by default */ + SnapshotWorkflow::RandomAccess); + break; + } + } + + switch (usedSnapshotWorkflow) + { + case SnapshotWorkflow::RandomAccess: { + return Snapshots(std::shared_ptr{ + new RandomAccessIteratorContainer(series.iterations)}); + } + case SnapshotWorkflow::Synchronous: { + std::function begin; + + if (access::write(IOHandler()->m_frontendAccess)) + { + begin = make_writing_stateful_iterator(*this, series); + } + else + { + begin = make_reading_stateful_iterator(*this, series); + } + return Snapshots(std::shared_ptr( + new StatefulSnapshotsContainer(std::move(begin)))); + } + } + throw std::runtime_error("unreachable!"); } void Series::parseBase() @@ -2949,16 +3206,15 @@ void Series::parseBase() WriteIterations Series::writeIterations() { - auto &series = get(); - if (!series.m_writeIterations.has_value()) + auto const access = IOHandler()->m_frontendAccess; + if (access != Access::CREATE && access != Access::APPEND) { - series.m_writeIterations = WriteIterations(this->iterations); - } - if (series.m_deferred_initialization.has_value()) - { - runDeferredInitialization(); + throw error::WrongAPIUsage( + "[Series::writeIterations()] May only be applied for access modes " + "CREATE or APPEND. Use Series::snapshots() for random-access-type " + "or for read-type workflows."); } - return series.m_writeIterations.value(); + return snapshots(SnapshotWorkflow::Synchronous); } void Series::close() diff --git a/src/WriteIterations.cpp b/src/WriteIterations.cpp deleted file mode 100644 index 0ae7246ae0..0000000000 --- a/src/WriteIterations.cpp +++ /dev/null @@ -1,124 +0,0 @@ -/* Copyright 2021 Franz Poeschel - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ - -#include "openPMD/WriteIterations.hpp" -#include "openPMD/Error.hpp" - -#include "openPMD/Series.hpp" - -namespace openPMD -{ -WriteIterations::SharedResources::SharedResources( - IterationsContainer_t _iterations) - : iterations(std::move(_iterations)) -{} - -WriteIterations::SharedResources::~SharedResources() -{ - if (auto IOHandler = iterations.IOHandler(); currentlyOpen.has_value() && - IOHandler && IOHandler->m_lastFlushSuccessful) - { - auto lastIterationIndex = currentlyOpen.value(); - auto &lastIteration = iterations.at(lastIterationIndex); - if (!lastIteration.closed()) - { - lastIteration.close(); - } - } -} - -WriteIterations::WriteIterations(IterationsContainer_t iterations) - : shared{std::make_shared>( - std::move(iterations))} -{} - -void WriteIterations::close() -{ - *shared = std::nullopt; -} - -WriteIterations::mapped_type &WriteIterations::operator[](key_type const &key) -{ - // make a copy - // explicit cast so MSVC can figure out how to do it correctly - return operator[](static_cast(key_type{key})); -} -WriteIterations::mapped_type &WriteIterations::operator[](key_type &&key) -{ - if (!shared || !shared->has_value()) - { - throw error::WrongAPIUsage( - "[WriteIterations] Trying to access after closing Series."); - } - auto &s = shared->value(); - auto lastIteration = currentIteration(); - if (lastIteration.has_value()) - { - auto lastIteration_v = lastIteration.value(); - if (lastIteration_v.iterationIndex == key) - { - return s.iterations.at(std::forward(key)); - } - else - { - lastIteration_v.close(); // continue below - } - } - s.currentlyOpen = key; - auto &res = s.iterations[std::forward(key)]; - if (res.getStepStatus() == StepStatus::NoStep) - { - try - { - res.beginStep(/* reread = */ false); - } - catch (error::OperationUnsupportedInBackend const &) - { - s.iterations.retrieveSeries() - .get() - .m_currentlyActiveIterations.clear(); - throw; - } - res.setStepStatus(StepStatus::DuringStep); - } - return res; -} - -std::optional WriteIterations::currentIteration() -{ - if (!shared || !shared->has_value()) - { - return std::nullopt; - } - auto &s = shared->value(); - if (!s.currentlyOpen.has_value()) - { - return std::nullopt; - } - Iteration ¤tIteration = s.iterations.at(s.currentlyOpen.value()); - if (currentIteration.closed()) - { - return std::nullopt; - } - return std::make_optional( - IndexedIteration(currentIteration, s.currentlyOpen.value())); -} -} // namespace openPMD diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index e7b48efddf..da9e09e2e0 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -39,7 +39,16 @@ namespace openPMD { namespace internal { - AttributableData::AttributableData() : m_writable{this} + SharedAttributableData::SharedAttributableData(AttributableData *attr) + : m_writable{attr} + {} + + AttributableData::AttributableData() + : SharedData_t(std::make_shared(this)) + {} + + AttributableData::AttributableData(SharedAttributableData *raw_ptr) + : SharedData_t({raw_ptr, [](auto const *) {}}) {} } // namespace internal @@ -53,7 +62,7 @@ Attributable::Attributable() } } -Attributable::Attributable(NoInit) +Attributable::Attributable(NoInit) noexcept {} Attribute Attributable::getAttribute(std::string const &key) const diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index ee15f2a1a0..948ff71580 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -19,10 +19,11 @@ * If not, see . */ #include "openPMD/backend/Writable.hpp" -#include "openPMD/Error.hpp" + #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/Series.hpp" -#include "openPMD/auxiliary/DerefDynamicCast.hpp" +#include "openPMD/backend/Attributable.hpp" + #include namespace openPMD diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index 59068c2728..d737dc8198 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -20,12 +20,16 @@ */ #include "openPMD/Series.hpp" #include "openPMD/IO/Access.hpp" +#include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/binding/python/Pickle.hpp" #include "openPMD/config.hpp" +#include "openPMD/snapshots/Snapshots.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/binding/python/Common.hpp" +#include #if openPMD_HAVE_MPI // re-implemented signatures: @@ -37,10 +41,10 @@ #include #include -struct SeriesIteratorPythonAdaptor : SeriesIterator +struct StatefulIteratorPythonAdaptor : LegacyIteratorAdaptor { - SeriesIteratorPythonAdaptor(SeriesIterator it) - : SeriesIterator(std::move(it)) + StatefulIteratorPythonAdaptor(LegacyIteratorAdaptor it) + : LegacyIteratorAdaptor(std::move(it)) {} /* @@ -71,13 +75,13 @@ not possible once it has been closed. )END") .def( "__getitem__", - [](WriteIterations writeIterations, Series::IterationIndex_t key) { + [](WriteIterations &writeIterations, Series::IterationIndex_t key) { auto lastIteration = writeIterations.currentIteration(); if (lastIteration.has_value() && - lastIteration.value().iterationIndex != key) + lastIteration.value()->first != key) { // this must happen under the GIL - lastIteration.value().close(); + lastIteration.value()->second.close(); } py::gil_scoped_release release; return writeIterations[key]; @@ -86,15 +90,26 @@ not possible once it has been closed. py::return_value_policy::copy) .def( "current_iteration", - &WriteIterations::currentIteration, + [](WriteIterations &writeIterations) + -> std::optional { + if (auto currentIteration = writeIterations.currentIteration(); + currentIteration.has_value()) + { + return IndexedIteration(**currentIteration); + } + else + { + return std::nullopt; + } + }, "Return the iteration that is currently being written to, if it " "exists."); - py::class_(m, "SeriesIterator") + py::class_(m, "StatefulIterator") .def( "__next__", - [](SeriesIteratorPythonAdaptor &iterator) { - if (iterator == SeriesIterator::end()) + [](StatefulIteratorPythonAdaptor &iterator) { + if (iterator == ReadIterations::end()) { throw py::stop_iteration(); } @@ -112,7 +127,7 @@ not possible once it has been closed. ++iterator; } iterator.first_iteration = false; - if (iterator == SeriesIterator::end()) + if (iterator == ReadIterations::end()) { throw py::stop_iteration(); } @@ -143,10 +158,10 @@ not possible once it has been closed. [](ReadIterations &readIterations) { // Simple iterator implementation: // But we need to release the GIL inside - // SeriesIterator::operator++, so manually it is + // StatefulIterator::operator++, so manually it is // return py::make_iterator( // readIterations.begin(), readIterations.end()); - return SeriesIteratorPythonAdaptor(readIterations.begin()); + return StatefulIteratorPythonAdaptor(readIterations.begin()); }, // keep handle alive while iterator exists py::keep_alive<0, 1>()); diff --git a/src/helper/list_series.cpp b/src/helper/list_series.cpp index eeb7523bb4..415f8e4743 100644 --- a/src/helper/list_series.cpp +++ b/src/helper/list_series.cpp @@ -113,10 +113,16 @@ std::ostream &listSeries(Series &series, bool const longer, std::ostream &out) if (longer) out << " all iterations: "; - for (auto const &i : series.readIterations()) + for (auto &[index, i] : series.snapshots()) { + // Necessary only if Series was opened in READ_RANDOM_ACCESS mode + // with `defer_iteration_parsing = true`. + if (!i.parsed()) + { + i.open(); + } if (longer) - out << i.iterationIndex << " "; + out << index << " "; // find unique record names std::transform( @@ -131,6 +137,7 @@ std::ostream &listSeries(Series &series, bool const longer, std::ostream &out) [](std::pair const &p) { return p.first; }); + i.close(); } if (longer) diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp new file mode 100644 index 0000000000..eb0bc54572 --- /dev/null +++ b/src/snapshots/ContainerImpls.cpp @@ -0,0 +1,393 @@ +#include "openPMD/snapshots/ContainerImpls.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/snapshots/ContainerTraits.hpp" +#include "openPMD/snapshots/IteratorHelpers.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" +#include +#include +#include + +namespace openPMD +{ +StatefulSnapshotsContainer::StatefulSnapshotsContainer( + std::function begin) + : members{std::move(begin)} +{} + +StatefulSnapshotsContainer::StatefulSnapshotsContainer( + StatefulSnapshotsContainer const &other) = default; +StatefulSnapshotsContainer::StatefulSnapshotsContainer( + StatefulSnapshotsContainer + &&other) noexcept(noexcept(Members(std::declval()))) = + default; +StatefulSnapshotsContainer &StatefulSnapshotsContainer::operator=( + StatefulSnapshotsContainer const &other) = default; +StatefulSnapshotsContainer &StatefulSnapshotsContainer:: +operator=(StatefulSnapshotsContainer &&other) noexcept(noexcept( + std::declval().operator=(std::declval()))) = default; + +auto StatefulSnapshotsContainer::get() -> StatefulIterator * +{ + if (!members.m_bufferedIterator.has_value()) + { + members.m_bufferedIterator = members.m_begin(); + } + return *members.m_bufferedIterator; +} +auto StatefulSnapshotsContainer::get() const -> StatefulIterator const * +{ + return members.m_bufferedIterator.value_or(nullptr); +} + +auto StatefulSnapshotsContainer::currentIteration() const + -> std::optional +{ + if (auto it = get(); it) + { + return it->peekCurrentlyOpenIteration(); + } + else + { + return std::nullopt; + } +} + +StatefulSnapshotsContainer::~StatefulSnapshotsContainer() = default; + +auto StatefulSnapshotsContainer::begin() -> iterator +{ + return stateful_to_opaque(*get()); +} +auto StatefulSnapshotsContainer::end() -> iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new StatefulIterator()}); +} +auto StatefulSnapshotsContainer::begin() const -> const_iterator +{ + throw error::WrongAPIUsage( + "[StatefulSnapshotsContainer::begin] Const iteration not possible on a " + "stateful container/iterator."); +} +auto StatefulSnapshotsContainer::end() const -> const_iterator +{ + throw error::WrongAPIUsage( + "[StatefulSnapshotsContainer::end] Const iteration not possible on a " + "stateful container/iterator."); +} +auto StatefulSnapshotsContainer::rbegin() -> iterator +{ + /* + * @todo maybe can adapt std::reverse_iterator as soon as the stateful + * iterator is powerful enough for this + */ + throw std::runtime_error( + "Reverse iteration not (yet) implemented on a stateful " + "container/iterator."); +} +auto StatefulSnapshotsContainer::rend() -> iterator +{ + throw std::runtime_error( + "Reverse iteration not (yet) implemented on a stateful " + "container/iterator."); +} +auto StatefulSnapshotsContainer::rbegin() const -> const_iterator +{ + throw error::WrongAPIUsage( + "[StatefulSnapshotsContainer::rbegin] Const iteration not possible on " + "a stateful container/iterator."); +} +auto StatefulSnapshotsContainer::rend() const -> const_iterator +{ + throw error::WrongAPIUsage( + "[StatefulSnapshotsContainer::rend] Const iteration not possible on a " + "stateful container/iterator."); +} + +bool StatefulSnapshotsContainer::empty() const +{ + auto it = get(); + return (!it) || !it->operator bool(); +} +auto StatefulSnapshotsContainer::size() const -> size_t +{ + /* + * This should return the sum over all IO steps, counting the number of + * snapshots contained in each step. This information should be tracked in + * future in order to have knowledge on where to find an Iteration once it + * was seen. + */ + throw std::runtime_error( + "[StatefulSnapshotsContainer::size()] Unimplemented"); +} + +auto StatefulSnapshotsContainer::at(key_type const &key) const + -> mapped_type const & +{ + auto it = get(); + auto current_iteration = it->peekCurrentlyOpenIteration(); + if (!current_iteration.has_value() || (*current_iteration)->first != key) + { + throw std::out_of_range( + "[StatefulSnapshotsContainer::at()] Cannot skip to a Snapshot that " + "is currently not active in a const context."); + } + return (*current_iteration)->second; +} +auto StatefulSnapshotsContainer::at(key_type const &key) -> mapped_type & +{ + auto base_iterator = get(); + auto &result = + base_iterator->seek({StatefulIterator::Seek::Seek_Iteration_t{key}}); + if (result.is_end()) + { + throw std::out_of_range( + "[StatefulSnapshotsContainer::at()] Cannot (yet) skip to " + "a Snapshot from an I/O step that is not active."); + } + return result->second; +} + +auto StatefulSnapshotsContainer::operator[](key_type const &key) + -> mapped_type & +{ + auto base_iterator = get(); + auto &shared = base_iterator->m_data; + if (!shared || !shared->has_value()) + { + throw error::WrongAPIUsage( + "[WriteIterations] Trying to access after closing Series."); + } + auto &s = shared->value(); + auto access = s.series.IOHandler()->m_frontendAccess; + + if (access == Access::READ_WRITE) + { + throw std::runtime_error("Stateful iteration on a read-write Series."); + } + if (access::write(access)) + { + auto lastIteration = base_iterator->peekCurrentlyOpenIteration(); + if (lastIteration.has_value()) + { + auto lastIteration_v = lastIteration.value(); + if (lastIteration_v->first == key) + { + return s.series.iterations.at(key); + } + else + { + lastIteration_v->second.close(); // continue below + } + } + if (auto it = s.series.iterations.find(key); + it == s.series.iterations.end()) + { + s.currentStep.map_during_t( + [&](detail::CurrentStep::During_t &during) { + ++during.step_count; + base_iterator->get().seen_iterations[key] = + during.step_count; + during.iteration_idx = key; + during.available_iterations_in_step = {key}; + }, + [&](detail::CurrentStep::AtTheEdge where_am_i) + -> detail::CurrentStep::During_t { + base_iterator->get().seen_iterations[key] = 0; + switch (where_am_i) + { + case detail::CurrentStep::AtTheEdge::Begin: + return detail::CurrentStep::During_t{0, key, {key}}; + case detail::CurrentStep::AtTheEdge::End: + throw error::Internal( + "Trying to create a new output step, but the " + "stream is " + "closed?"); + } + throw std::runtime_error("Unreachable!"); + }); + } + auto &res = s.series.iterations[key]; + if (res.getStepStatus() != StepStatus::DuringStep) + { + try + { + res.beginStep(/* reread = */ false); + } + catch (error::OperationUnsupportedInBackend const &) + { + s.series.iterations.retrieveSeries() + .get() + .m_currentlyActiveIterations.clear(); + throw; + } + res.setStepStatus(StepStatus::DuringStep); + } + return res; + } + else if (access::read(access)) + { + auto &result = base_iterator->seek( + {StatefulIterator::Seek::Seek_Iteration_t{key}}); + if (result.is_end()) + { + throw std::out_of_range( + "[StatefulSnapshotsContainer::operator[]()] Cannot (yet) skip " + "to a Snapshot from an I/O step that is not active."); + } + return result->second; + } + throw error::Internal("Control flow error: This should be unreachable."); +} + +auto StatefulSnapshotsContainer::clear() -> void +{ + throw std::runtime_error( + "[StatefulSnapshotsContainer::clear()] Unimplemented"); +} + +auto StatefulSnapshotsContainer::find(key_type const &) -> iterator +{ + throw error::WrongAPIUsage( + "[StatefulSnapshotsContainer::find] `find()` not available in stateful " + "iteration as there is only one shared iterator per Series and " + "`find()` would need to modify that."); +} +auto StatefulSnapshotsContainer::find(key_type const &) const -> const_iterator +{ + throw error::WrongAPIUsage( + "[StatefulSnapshotsContainer::find] `find()` not available in stateful " + "iteration as there is only one shared iterator per Series and " + "`find()` would need to modify that."); +} + +auto StatefulSnapshotsContainer::contains(key_type const &) const -> bool +{ + throw std::runtime_error("Unimplemented"); +} + +RandomAccessIteratorContainer::RandomAccessIteratorContainer( + Container cont) + : m_cont(std::move(cont)) +{} + +RandomAccessIteratorContainer::~RandomAccessIteratorContainer() = default; + +RandomAccessIteratorContainer::RandomAccessIteratorContainer( + RandomAccessIteratorContainer const &other) = default; +RandomAccessIteratorContainer::RandomAccessIteratorContainer( + RandomAccessIteratorContainer &&other) noexcept = default; +RandomAccessIteratorContainer &RandomAccessIteratorContainer::operator=( + RandomAccessIteratorContainer const &other) = default; +RandomAccessIteratorContainer &RandomAccessIteratorContainer::operator=( + RandomAccessIteratorContainer &&other) noexcept = default; + +auto RandomAccessIteratorContainer::currentIteration() const + -> std::optional +{ + if (auto begin = m_cont.begin(); begin != m_cont.end()) + { + return std::make_optional(&*begin); + } + else + { + return std::nullopt; + } +} + +auto RandomAccessIteratorContainer::begin() -> iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.begin())}); +} +auto RandomAccessIteratorContainer::end() -> iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.end())}); +} +auto RandomAccessIteratorContainer::begin() const -> const_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.begin())}); +} +auto RandomAccessIteratorContainer::end() const -> const_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.end())}); +} +auto RandomAccessIteratorContainer::rbegin() -> reverse_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.rbegin())}); +} +auto RandomAccessIteratorContainer::rend() -> reverse_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.end())}); +} +auto RandomAccessIteratorContainer::rbegin() const -> const_reverse_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.rbegin())}); +} +auto RandomAccessIteratorContainer::rend() const -> const_reverse_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.rend())}); +} + +auto RandomAccessIteratorContainer::empty() const -> bool +{ + return m_cont.empty(); +} +auto RandomAccessIteratorContainer::size() const -> size_t +{ + return m_cont.size(); +} + +auto RandomAccessIteratorContainer::at(key_type const &key) const + -> mapped_type const & +{ + return m_cont.at(key); +} + +auto RandomAccessIteratorContainer::operator[](key_type const &key) + -> mapped_type & +{ + return m_cont[key]; +} + +auto RandomAccessIteratorContainer::clear() -> void +{ + throw std::runtime_error("Unimplemented"); +} + +auto RandomAccessIteratorContainer::find(key_type const &key) -> iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.find(key))}); +} +auto RandomAccessIteratorContainer::find(key_type const &key) const + -> const_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.find(key))}); +} + +auto RandomAccessIteratorContainer::contains(key_type const &key) const -> bool +{ + return m_cont.contains(key); +} +} // namespace openPMD diff --git a/src/snapshots/ContainerTraits.cpp b/src/snapshots/ContainerTraits.cpp new file mode 100644 index 0000000000..01fa7b53cb --- /dev/null +++ b/src/snapshots/ContainerTraits.cpp @@ -0,0 +1,114 @@ +#include "openPMD/snapshots/ContainerTraits.hpp" +#include "openPMD/Iteration.hpp" +#include + +namespace openPMD +{ +// constructors +template +OpaqueSeriesIterator::OpaqueSeriesIterator( + std::unique_ptr> internal_iterator) + : m_internal_iterator(std::move(internal_iterator)) +{} + +// copy/move constructor +template +OpaqueSeriesIterator::OpaqueSeriesIterator( + OpaqueSeriesIterator const &other) + : m_internal_iterator(other.m_internal_iterator->clone()) +{} +template +OpaqueSeriesIterator::OpaqueSeriesIterator( + OpaqueSeriesIterator &&other) noexcept = default; +template + +// copy/move assignment +OpaqueSeriesIterator & +OpaqueSeriesIterator::operator=(OpaqueSeriesIterator const &other) +{ + m_internal_iterator = other.m_internal_iterator->clone(); + return *this; +} +template +OpaqueSeriesIterator &OpaqueSeriesIterator::operator=( + OpaqueSeriesIterator &&other) noexcept = default; + +// destructor +template +OpaqueSeriesIterator::~OpaqueSeriesIterator() = default; + +// dereference +template +auto OpaqueSeriesIterator::operator*() -> value_type & +{ + return m_internal_iterator->dereference_operator(); +} +template +auto OpaqueSeriesIterator::operator*() const -> value_type const & +{ + return m_internal_iterator->dereference_operator(); +} + +// increment/decrement +template +auto OpaqueSeriesIterator::operator++() -> OpaqueSeriesIterator & +{ + m_internal_iterator->increment_operator(); + return *this; +} +template +auto OpaqueSeriesIterator::operator--() -> OpaqueSeriesIterator & +{ + m_internal_iterator->decrement_operator(); + return *this; +} +template +auto OpaqueSeriesIterator::operator++(int) -> OpaqueSeriesIterator +{ + auto prev = *this; + ++(*this); + return prev; +} +template +auto OpaqueSeriesIterator::operator--(int) -> OpaqueSeriesIterator +{ + auto prev = *this; + --(*this); + return prev; +} + +// comparison +template +auto OpaqueSeriesIterator::operator==( + OpaqueSeriesIterator const &other) const -> bool +{ + return m_internal_iterator->equality_operator(*other.m_internal_iterator); +} + +using value_type = + Container::value_type; +template class OpaqueSeriesIterator; +template class OpaqueSeriesIterator; + +AbstractSnapshotsContainer::~AbstractSnapshotsContainer() = default; + +auto AbstractSnapshotsContainer::currentIteration() + -> std::optional +{ + if (auto maybe_value = static_cast(this) + ->currentIteration(); + maybe_value.has_value()) + { + return {const_cast(*maybe_value)}; + } + else + { + return std::nullopt; + } +} +auto AbstractSnapshotsContainer::at(key_type const &key) -> mapped_type & +{ + return const_cast( + static_cast(this)->at(key)); +} +} // namespace openPMD diff --git a/src/snapshots/IteratorHelpers.cpp b/src/snapshots/IteratorHelpers.cpp new file mode 100644 index 0000000000..0621fdb71a --- /dev/null +++ b/src/snapshots/IteratorHelpers.cpp @@ -0,0 +1,15 @@ +#include "openPMD/snapshots/IteratorHelpers.hpp" + +namespace openPMD +{ +using value_type = + Container::value_type; +auto stateful_to_opaque(StatefulIterator const &it) + -> OpaqueSeriesIterator +{ + std::unique_ptr> internal_iterator_cloned{ + new StatefulIterator(it)}; + return OpaqueSeriesIterator( + std::move(internal_iterator_cloned)); +} +} // namespace openPMD diff --git a/src/snapshots/IteratorTraits.cpp b/src/snapshots/IteratorTraits.cpp new file mode 100644 index 0000000000..b8d18659d1 --- /dev/null +++ b/src/snapshots/IteratorTraits.cpp @@ -0,0 +1,153 @@ +#include "openPMD/snapshots/IteratorTraits.hpp" +#include "openPMD/snapshots/RandomAccessIterator.hpp" +#include "openPMD/snapshots/Snapshots.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" +#include +#include + +namespace openPMD +{ +template +DynamicSeriesIterator::~DynamicSeriesIterator() = default; + +template +auto DynamicSeriesIterator::dereference_operator() -> value_type & +{ + return const_cast( + static_cast(this) + ->dereference_operator()); +} + +template +AbstractSeriesIterator::~AbstractSeriesIterator() = + default; + +// dereference +template +auto AbstractSeriesIterator::operator->() const + -> value_type const * +{ + return &this_child()->operator*(); +} +template +auto AbstractSeriesIterator::operator->() + -> value_type * +{ + return &this_child()->operator*(); +} + +// increment/decrement +template +ChildClass +AbstractSeriesIterator::default_increment_operator(int) +{ + auto prev = *this_child(); + this_child()->operator++(); + return prev; +} +template +ChildClass +AbstractSeriesIterator::default_decrement_operator(int) +{ + auto prev = *this_child(); + this_child()->operator--(); + return prev; +} + +// comparison +template +bool AbstractSeriesIterator::operator!=( + ChildClass const &other) const +{ + return !this_child()->operator==(other); +} + +// helpers +template +ChildClass *AbstractSeriesIterator::this_child() +{ + return static_cast(this); +} +template +ChildClass const * +AbstractSeriesIterator::this_child() const +{ + return static_cast(this); +} + +// out-of-line arithmetic operators +template +ChildClass operator+( + Iteration::IterationIndex_t index, + AbstractSeriesIterator const &iterator) +{ + return static_cast(iterator).operator+(index); +} + +/************* + * overrides * + *************/ + +// dereference +template +auto AbstractSeriesIterator::dereference_operator() + const -> value_type const & +{ + return this_child()->operator*(); +} + +// increment/decrement +template +auto AbstractSeriesIterator::increment_operator() + -> parent_t & +{ + return ++*this_child(); +} +template +auto AbstractSeriesIterator::decrement_operator() + -> parent_t & +{ + return --*this_child(); +} + +// comparison +template +bool AbstractSeriesIterator::equality_operator( + parent_t const &other) const +{ + if (auto child = dynamic_cast(&other); child) + { + return this_child()->operator==(*child); + } + else + { + return false; // or throw error? + } +} + +template +auto AbstractSeriesIterator::clone() const + -> std::unique_ptr +{ + return std::unique_ptr( + new ChildClass(*static_cast(this))); +} + +using iterations_container_t = + Container; + +#define OPENPMD_INSTANTIATE(type) \ + template class AbstractSeriesIterator; +OPENPMD_INSTANTIATE(StatefulIterator) +OPENPMD_INSTANTIATE(OpaqueSeriesIterator) +OPENPMD_INSTANTIATE( + OpaqueSeriesIterator) +OPENPMD_INSTANTIATE(RandomAccessIterator) +OPENPMD_INSTANTIATE( + RandomAccessIterator) +OPENPMD_INSTANTIATE( + RandomAccessIterator) +OPENPMD_INSTANTIATE( + RandomAccessIterator) +#undef OPENPMD_INSTANTIATE +} // namespace openPMD diff --git a/src/snapshots/RandomAccessIterator.cpp b/src/snapshots/RandomAccessIterator.cpp new file mode 100644 index 0000000000..689c675db7 --- /dev/null +++ b/src/snapshots/RandomAccessIterator.cpp @@ -0,0 +1,86 @@ +#include "openPMD/snapshots/RandomAccessIterator.hpp" +namespace openPMD +{ +template +inline RandomAccessIterator::RandomAccessIterator(iterator_t it) + : m_it(it) +{} + +template +RandomAccessIterator::~RandomAccessIterator() = default; + +template +RandomAccessIterator::RandomAccessIterator( + RandomAccessIterator const &other) = default; +template +RandomAccessIterator::RandomAccessIterator( + RandomAccessIterator + &&other) noexcept(noexcept(iterator_t(std::declval()))) = + default; +template +RandomAccessIterator &RandomAccessIterator::operator=( + RandomAccessIterator const &other) = default; +template +RandomAccessIterator &RandomAccessIterator::operator=( + RandomAccessIterator + &&other) noexcept(noexcept(std::declval(). + operator=(std::declval()))) = + default; + +template +auto RandomAccessIterator::operator*() -> value_type & +{ + return *m_it; +} + +template +auto RandomAccessIterator::operator*() const -> value_type const & +{ + return *m_it; +} + +template +auto RandomAccessIterator::operator++() -> RandomAccessIterator & +{ + ++m_it; + return *this; +} + +template +auto RandomAccessIterator::operator--() -> RandomAccessIterator & +{ + --m_it; + return *this; +} + +template +auto RandomAccessIterator::operator++(int i) -> RandomAccessIterator +{ + return parent_t::default_increment_operator(i); +} + +template +auto RandomAccessIterator::operator--(int i) -> RandomAccessIterator +{ + return parent_t::default_decrement_operator(i); +} + +template +auto RandomAccessIterator::operator==( + RandomAccessIterator const &other) const -> bool +{ + return m_it == other.m_it; +} + +using iterator = Container::iterator; +using const_iterator = + Container::const_iterator; +using reverse_iterator = + Container::reverse_iterator; +using const_reverse_iterator = + Container::const_reverse_iterator; +template class RandomAccessIterator; +template class RandomAccessIterator; +template class RandomAccessIterator; +template class RandomAccessIterator; +} // namespace openPMD diff --git a/src/snapshots/Snapshots.cpp b/src/snapshots/Snapshots.cpp new file mode 100644 index 0000000000..e535afffbd --- /dev/null +++ b/src/snapshots/Snapshots.cpp @@ -0,0 +1,104 @@ +#include "openPMD/snapshots/Snapshots.hpp" +namespace openPMD +{ +Snapshots::Snapshots(std::shared_ptr snapshots) + : m_snapshots(std::move(snapshots)) +{} +inline auto Snapshots::get() const -> AbstractSnapshotsContainer const & +{ + return *m_snapshots; +} +inline auto Snapshots::get() -> AbstractSnapshotsContainer & +{ + return *m_snapshots; +} +auto Snapshots::currentIteration() -> std::optional +{ + return get().currentIteration(); +} +auto Snapshots::currentIteration() const -> std::optional +{ + return get().currentIteration(); +} +auto Snapshots::begin() -> iterator +{ + return get().begin(); +} +auto Snapshots::end() -> iterator +{ + return get().end(); +} +auto Snapshots::begin() const -> const_iterator +{ + return static_cast(*m_snapshots) + .begin(); +} +auto Snapshots::end() const -> const_iterator +{ + return static_cast(*m_snapshots).end(); +} +auto Snapshots::rbegin() -> reverse_iterator +{ + return get().begin(); +} +auto Snapshots::rend() -> reverse_iterator +{ + return get().end(); +} +auto Snapshots::rbegin() const -> const_reverse_iterator +{ + return static_cast(*m_snapshots) + .begin(); +} +auto Snapshots::rend() const -> const_reverse_iterator +{ + return get().end(); +} + +auto Snapshots::empty() const -> bool +{ + return get().empty(); +} +auto Snapshots::size() const -> size_t +{ + return get().size(); +} + +auto Snapshots::at(key_type const &key) const -> mapped_type const & +{ + return get().at(key); +} +auto Snapshots::at(key_type const &key) -> mapped_type & +{ + return get().at(key); +} +auto Snapshots::operator[](key_type const &key) -> mapped_type & +{ + return get().operator[](key); +} + +auto Snapshots::clear() -> void +{ + return get().clear(); +} + +auto Snapshots::find(key_type const &key) -> iterator +{ + return get().find(key); +} +auto Snapshots::find(key_type const &key) const -> const_iterator +{ + return get().find(key); +} + +auto Snapshots::count(key_type const &key) const -> size_t +{ + return contains(key) ? 1 : 0; +} + +auto Snapshots::contains(key_type const &key) const -> bool +{ + return get().contains(key); +} + +} // namespace openPMD diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp new file mode 100644 index 0000000000..7db8cf5d44 --- /dev/null +++ b/src/snapshots/StatefulIterator.cpp @@ -0,0 +1,964 @@ +/* Copyright 2021 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/snapshots/StatefulIterator.hpp" +#include "openPMD/Datatype.hpp" +#include "openPMD/Error.hpp" + +#include "openPMD/Iteration.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/auxiliary/Variant.hpp" +#include "openPMD/backend/ParsePreference.hpp" + +#include +#include +#include +#include +#include +#include + +namespace openPMD +{ + +namespace detail +{ + step_status_types::During_t::During_t( + size_t step_count_in, + std::optional iteration_idx_in, + std::vector + available_iterations_in_step_in) + : step_count(step_count_in) + , iteration_idx(iteration_idx_in) + , available_iterations_in_step( + std::move(available_iterations_in_step_in)) + {} + + template + auto CurrentStep::get_variant() -> std::optional + { + auto res = std::get_if(this); + if (res) + { + return std::make_optional(res); + } + else + { + return std::nullopt; + } + } + + template + [[nodiscard]] auto CurrentStep::get_variant() const + -> std::optional + { + auto res = std::get_if(*this); + if (res) + { + return {res}; + } + else + { + return std::nullopt; + } + } + + auto CurrentStep::get_iteration_index() const + -> std::optional + { + using res_t = std::optional; + return std::visit( + auxiliary::overloaded{ + [](auto const &) -> res_t { return std::nullopt; }, + [](During_t const &during) -> res_t { + return during.iteration_idx; + }}, + this->as_base()); + } + +} // namespace detail + +StatefulIterator::SharedData::~SharedData() +{ + // debugging block +#if 0 + std::map tmp; + std::copy( + seen_iterations.begin(), + seen_iterations.end(), + std::inserter(tmp, tmp.begin())); + std::cout << "SEEN ITERATIONS:\n"; + for(auto [it, step]: tmp) + { + std::cout << '\t' << it << ":\t" << step << '\n'; + } + std::cout << std::endl; +#endif + auto IOHandler = series.IOHandler(); + auto current_iteration = currentIteration(); + if (IOHandler && current_iteration.has_value() && IOHandler && + IOHandler->m_lastFlushSuccessful) + { + auto lastIterationIndex = *current_iteration; + if (!series.iterations.contains(*current_iteration)) + { + return; + } + auto &lastIteration = series.iterations.at(lastIterationIndex); + if (!lastIteration.closed()) + { + lastIteration.close(); + } + } +} + +auto StatefulIterator::SharedData::currentIteration() const + -> std::optional +{ + return currentStep.get_iteration_index(); +} + +namespace +{ + bool reread(std::optional parsePreference) + { + if (parsePreference.has_value()) + { + using PP = Parameter::ParsePreference; + + switch (parsePreference.value()) + { + case PP::PerStep: + return true; + case PP::UpFront: + return false; + } + return false; + } + else + { + throw error::Internal( + "Group/Variable-based encoding: Parse preference must be set."); + } + } +} // namespace + +StatefulIterator::StatefulIterator() = default; +StatefulIterator::~StatefulIterator() = default; + +StatefulIterator::StatefulIterator(StatefulIterator const &other) = default; +StatefulIterator::StatefulIterator(StatefulIterator &&other) noexcept = default; +StatefulIterator & +StatefulIterator::operator=(StatefulIterator const &other) = default; +StatefulIterator & +StatefulIterator::operator=(StatefulIterator &&other) noexcept = default; + +auto StatefulIterator::get() -> SharedData & +{ + return m_data->value(); +} +auto StatefulIterator::get() const -> SharedData const & +{ + return m_data->value(); +} + +void StatefulIterator::initSeriesInLinearReadMode() +{ + auto &data = get(); + auto &series = data.series; + series.IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing; + try + { + switch (series.iterationEncoding()) + { + using IE = IterationEncoding; + case IE::fileBased: + series.readFileBased(std::nullopt); + break; + case IE::groupBased: + case IE::variableBased: { + Parameter fOpen; + fOpen.name = series.get().m_name; + series.IOHandler()->enqueue(IOTask(&series, fOpen)); + series.IOHandler()->flush(internal::defaultFlushParams); + using PP = Parameter::ParsePreference; + switch (*fOpen.out_parsePreference) + { + case PP::PerStep: + series.advance(AdvanceMode::BEGINSTEP); + series.readGorVBased( + /* do_always_throw_errors = */ false, + /* init = */ true, + std::nullopt); + break; + case PP::UpFront: + series.readGorVBased( + /* do_always_throw_errors = */ false, + /* init = */ true, + std::nullopt); + /* + * In linear read mode (where no parsing at all is done upon + * constructing the Series), it might turn out after parsing + * that what we expected to be a group-based Series was in fact + * a single file of a file-based Series. + * (E.g. when opening "data00000100.h5" directly instead of + * "data%T.h5") + * So we need to check the current value of + * `iterationEncoding()` once more. + */ + if (series.iterationEncoding() != IterationEncoding::fileBased) + { + series.advance(AdvanceMode::BEGINSTEP); + } + break; + } + data.parsePreference = *fOpen.out_parsePreference; + break; + } + } + } + catch (...) + { + series.IOHandler()->m_seriesStatus = internal::SeriesStatus::Default; + throw; + } + series.IOHandler()->m_seriesStatus = internal::SeriesStatus::Default; +} + +void StatefulIterator::close() +{ + *m_data = std::nullopt; // turn this into end iterator +} + +auto StatefulIterator::turn_into_end_iterator(TypeOfEndIterator type) -> void +{ + auto &data = get(); + switch (type) + { + case TypeOfEndIterator::NoMoreSteps: + data.currentStep = CurrentStep::After; + break; + case TypeOfEndIterator::NoMoreIterationsInStep: + data.currentStep.map_during_t( + [](CurrentStep::During_t &during) { + during.iteration_idx = std::nullopt; + }, + [](auto const &) { + return CurrentStep::During_t{0, std::nullopt, {}}; + } + + ); + break; + } +} + +namespace +{ + auto restrict_to_unseen_iterations( + std::vector &indexes, + std::unordered_map + &seen_iterations, + size_t insert_into_step) -> void + { + for (auto vec_it = indexes.rbegin(); vec_it != indexes.rend();) + { + auto map_it = seen_iterations.find(*vec_it); + if (map_it == seen_iterations.end()) + { + seen_iterations.emplace_hint(map_it, *vec_it, insert_into_step); + ++vec_it; + } + else + { + ++vec_it; + // sic! base() refers to the next element... + // erase() only invalidates iterators to the right + // (this is why this iterates in reverse) + indexes.erase(vec_it.base()); + } + } + } +} // namespace + +auto StatefulIterator::resetCurrentIterationToBegin( + size_t num_skipped_iterations, std::vector indexes) + -> void +{ + auto &data = get(); + data.currentStep.map_during_t( + [&](CurrentStep::During_t &during) { + during.step_count += num_skipped_iterations; + restrict_to_unseen_iterations( + indexes, data.seen_iterations, during.step_count); + during.available_iterations_in_step = std::move(indexes); + if (during.available_iterations_in_step.empty()) + { + during.iteration_idx = std::nullopt; + } + else + { + during.iteration_idx = + *during.available_iterations_in_step.begin(); + } + }, + [&](CurrentStep::AtTheEdge whereAmI) + -> std::optional { + switch (whereAmI) + { + case detail::CurrentStep::AtTheEdge::Begin: { + restrict_to_unseen_iterations( + indexes, data.seen_iterations, num_skipped_iterations); + if (indexes.empty()) + { + return std::nullopt; + } + auto first_iteration = *indexes.begin(); + // Begin iterating + return detail::CurrentStep::During_t{ + num_skipped_iterations, + first_iteration, + std::move(indexes)}; + } + case detail::CurrentStep::AtTheEdge::End: + return std::nullopt; + } + throw std::runtime_error("Unreachable!"); + }); +} + +auto StatefulIterator::peekCurrentlyOpenIteration() const + -> std::optional +{ + if (!m_data || !m_data->has_value()) + { + return std::nullopt; + } + auto &s = m_data->value(); + auto const &maybeCurrentIteration = s.currentIteration(); + if (!maybeCurrentIteration.has_value()) + { + return std::nullopt; + } + auto currentIteration = s.series.iterations.find(*maybeCurrentIteration); + if (currentIteration == s.series.iterations.end()) + { + return std::nullopt; + } + if (currentIteration->second.closed()) + { + return std::nullopt; + } + return std::make_optional(&*currentIteration); +} +auto StatefulIterator::peekCurrentlyOpenIteration() + -> std::optional +{ + if (auto res = static_cast(this) + ->peekCurrentlyOpenIteration(); + res.has_value()) + { + return {const_cast(*res)}; + } + else + { + return std::nullopt; + } +} + +auto StatefulIterator::reparse_possibly_deleted_iteration(iteration_index_t idx) + -> void +{ + auto &data = get(); + if (!data.series.iterations.contains(idx)) + { + withRWAccess(data.series.IOHandler()->m_seriesStatus, [&]() { + switch (data.series.iterationEncoding()) + { + + case IterationEncoding::fileBased: + data.series.readFileBased({idx}); + break; + case IterationEncoding::groupBased: + case IterationEncoding::variableBased: + data.series.readGorVBased( + /* do_always_throw_errors = */ true, + /* init = */ false, + {idx}); + break; + } + }); + } +} + +StatefulIterator::StatefulIterator(tag_write_t, Series const &series_in) + : m_data{std::make_shared>(std::in_place)} +{ + auto &data = get(); + /* + * Since the iterator is stored in + * internal::SeriesData::m_sharedStatefulIterator, + * we need to use a non-owning Series instance here for tie-breaking + * purposes. + * This is ok due to the usual C++ iterator invalidation workflows + * (deleting the original container invalidates the iterator). + */ + data.series = series_in.m_attri->asInternalCopyOf(); +} + +StatefulIterator::StatefulIterator( + tag_read_t, + Series const &series_in, + std::optional const &parsePreference) + : m_data{std::make_shared>(std::in_place)} +{ + auto &data = get(); + data.parsePreference = parsePreference; + /* + * Since the iterator is stored in + * internal::SeriesData::m_sharedStatefulIterator, + * we need to use a non-owning Series instance here for tie-breaking + * purposes. + * This is ok due to the usual C++ iterator invalidation workflows + * (deleting the original container invalidates the iterator). + */ + data.series = series_in.m_attri->asInternalCopyOf(); + + auto &series = data.series; + if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR && + series.iterations.empty()) + { + initSeriesInLinearReadMode(); + } + + switch (series.iterationEncoding()) + { + case IterationEncoding::fileBased: { + initIteratorFilebased(); + break; + } + case IterationEncoding::groupBased: + case IterationEncoding::variableBased: + if (!seek({Seek::Next})) + { + throw std::runtime_error("Must not happen"); + } + break; + } +} + +std::optional StatefulIterator::nextIterationInStep() +{ + auto &data = get(); + auto maybeCurrentIteration = + data.currentStep.get_variant(); + + if (!maybeCurrentIteration.has_value()) + { + return std::nullopt; + } + CurrentStep::During_t ¤tIteration = **maybeCurrentIteration; + + auto no_result = [&]() { + currentIteration.iteration_idx = std::nullopt; + return std::nullopt; + }; + + if (!currentIteration.iteration_idx.has_value()) + { + return no_result(); + } + + auto ¤t_iteration_idx = *currentIteration.iteration_idx; + + if (auto it = std::find( + currentIteration.available_iterations_in_step.begin(), + currentIteration.available_iterations_in_step.end(), + current_iteration_idx); + it != currentIteration.available_iterations_in_step.end()) + { + ++it; + if (it == currentIteration.available_iterations_in_step.end()) + { + return no_result(); + } + current_iteration_idx = *it; + } + else + { + return no_result(); + } + + auto &series = data.series; + + try + { + reparse_possibly_deleted_iteration(current_iteration_idx); + series.iterations.at(current_iteration_idx).open(); + } + catch (error::ReadError const &err) + { + std::cerr << "[StatefulIterator] Cannot read iteration '" + << *maybeCurrentIteration + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + return nextIterationInStep(); + } + + return {this}; +} + +auto StatefulIterator::skipToIterationInStep(Iteration::IterationIndex_t idx) + -> std::optional +{ + auto &data = get(); + auto maybeCurrentIteration = + data.currentStep.get_variant(); + + if (!maybeCurrentIteration.has_value()) + { + return std::nullopt; + } + CurrentStep::During_t ¤tIteration = **maybeCurrentIteration; + + if (std::find( + currentIteration.available_iterations_in_step.begin(), + currentIteration.available_iterations_in_step.end(), + idx) == currentIteration.available_iterations_in_step.end()) + { + return std::nullopt; + } + + // This is called upon user request, i.e. ReadErrors should be caught in + // user code + reparse_possibly_deleted_iteration(idx); + data.series.iterations.at(idx).open(); + currentIteration.iteration_idx = idx; + return {this}; +} + +std::optional +StatefulIterator::nextStep(size_t recursion_depth) +{ + auto &data = get(); + std::vector current_iterations; + // since we are in group-based iteration layout, it does not + // matter which iteration we begin a step upon + AdvanceStatus status{}; + try + { + std::tie(status, current_iterations) = Iteration::beginStep( + {}, + data.series, + /* reread = */ reread(data.parsePreference)); + } + catch (error::ReadError const &err) + { + std::cerr << "[StatefulIterator] Cannot read iteration due to error " + "below, will skip it.\n" + << err.what() << std::endl; + data.series.advance(AdvanceMode::ENDSTEP); + return nextStep(recursion_depth + 1); + } + + bool close = [&]() { + switch (status) + { + + case AdvanceStatus::OK: + return false; + case AdvanceStatus::OVER: + return true; + case AdvanceStatus::RANDOMACCESS: + return std::visit( + auxiliary::overloaded{ + [](CurrentStep::Before_t const &) { return false; }, + [](auto const &) { return true; }}, + data.currentStep.as_base()); + } + throw std::runtime_error("Unreachable!"); + }(); + + if (close) + { + this->turn_into_end_iterator(TypeOfEndIterator::NoMoreSteps); + } + else + { + if (auto during = data.currentStep.get_variant(); + during.has_value()) + { + ++(*during)->step_count; + } + resetCurrentIterationToBegin( + recursion_depth, std::move(current_iterations)); + } + return {this}; +} + +std::optional StatefulIterator::loopBody(Seek const &seek) +{ + auto &data = get(); + Series &series = data.series; + auto &iterations = series.iterations; + + { /* + * Might not be present because parsing might have failed in previous step + */ + auto maybe_current_iteration = data.currentStep.get_iteration_index(); + if (maybe_current_iteration.has_value() && + // don't deactivate the iteration if it's the one that's currently + // active anyway + std::visit( + auxiliary::overloaded{ + [&](detail::seek_types::Next_t const &) { return true; }, + [&](detail::seek_types::Seek_Iteration_t const + &go_to_iteration) { + return go_to_iteration.iteration_idx != + *maybe_current_iteration; + }}, + seek.as_base()) && + // Iteration might have previously been deleted + iterations.contains(*maybe_current_iteration)) + { + auto ¤tIteration = iterations.at(*maybe_current_iteration); + if (!currentIteration.closed()) + { + currentIteration.close(); + } + // sic! only erase if the Iteration was explicitly closed from user + // code, don't implicitly sweep the iteration away from under the + // user + else if ( + series.IOHandler()->m_frontendAccess == Access::READ_LINEAR) + { + data.series.iterations.container().erase( + *maybe_current_iteration); + } + } + } + + /* + * Some checks and postprocessing: + * + * 1. Do some correctness checks. + * 2. If no data or the end Iterator is returned, then do nothing. + * 3. If the current step has no further data, then end the step and return + * a nullopt. + * 4. If an Iteration is successfully returned, check its close status, + * maybe open it or throw an error if an unexpected status is found. + */ + auto guardReturn = + [&series, &iterations, &data]( + auto const &option) -> std::optional { + if (!option.has_value() || + std::holds_alternative( + (*option)->get().currentStep)) + { + return option; + } + /* + * A step was successfully opened, but no iterations are contained. + * This might happen when iterations from the step are ignored, e.g. + * a duplicate iteration has been written by Append mode. + */ + auto maybe_current_iteration = data.currentStep.get_iteration_index(); + if (!maybe_current_iteration.has_value()) + { + series.advance(AdvanceMode::ENDSTEP); + return std::nullopt; + } + auto ¤t_iteration = *maybe_current_iteration; + + if (!iterations.contains(current_iteration)) + { + throw error::Internal( + "[StatefulIterator::loopBody] An iteration index was returned, " + "but the corresponding Iteration does not exist."); + } + + auto iteration = iterations.at(current_iteration); + switch (iteration.get().m_closed) + { + case internal::CloseStatus::ParseAccessDeferred: + try + { + iterations.at(current_iteration).open(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read iteration '" << current_iteration + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + option.value()->deactivateDeadIteration(current_iteration); + return std::nullopt; + } + [[fallthrough]]; + case internal::CloseStatus::Open: + return option; + case internal::CloseStatus::Closed: + case internal::CloseStatus::ClosedInFrontend: + throw error::Internal( + "[StatefulIterator] Next step returned an iteration that " + "is already closed, should have opened it."); + } + throw std::runtime_error("Unreachable!"); + }; + + /* + * First check if the requested Iteration can be found within the current + * step. + */ + { + std::optional optionallyAnIteration = std::visit( + auxiliary::overloaded{ + [&](Seek::Next_t) { return nextIterationInStep(); }, + [&](Seek::Seek_Iteration_t const &skip_to_iteration) { + return skipToIterationInStep( + skip_to_iteration.iteration_idx); + }}, + seek.as_base()); + if (optionallyAnIteration.has_value()) + { + return guardReturn(optionallyAnIteration); + } + } + + /* + * The current step does not contain the requested data. In file-based + * iteration encoding or in UpFront parse mode (i.e. no steps), this means + * that we've reached the end now. + */ + + if (series.iterationEncoding() == IterationEncoding::fileBased || + (data.parsePreference == internal::ParsePreference::UpFront && + // If the step status is still Begin, the Series is currently being + // initiated. In group-/variable-based encoding, a step needs to be + // begun at least once. This is because a Series with + // ParsePreference::UpFront is modeled as a Series with one big step + // that contains all Iterations. + std::holds_alternative( + data.currentStep.as_base()))) + { + // this one is handled above, stream is over once it proceeds to here + this->turn_into_end_iterator(TypeOfEndIterator::NoMoreIterationsInStep); + return {this}; + } + + /* + * The currently active iterations have been exhausted. + * Now see if there are further iterations to be found in next step. + */ + auto option = std::visit( + auxiliary::overloaded{ + [&](Seek::Next_t) { return nextStep(/* recursion_depth = */ 0); }, + [](Seek::Seek_Iteration_t const &skip_to_iteration) + -> std::optional { + throw std::runtime_error( + "[StatefulIterator] Skipping to iteration " + + std::to_string(skip_to_iteration.iteration_idx) + + "from another step not supported yet"); + }}, + seek.as_base()); + return guardReturn(option); +} + +void StatefulIterator::initIteratorFilebased() +{ + auto &data = get(); + auto &series = data.series; + if (series.iterations.empty()) + { + this->turn_into_end_iterator(TypeOfEndIterator::NoMoreIterationsInStep); + return; + } + std::vector indexes; + indexes.reserve(series.iterations.size()); + std::transform( + series.iterations.begin(), + series.iterations.end(), + std::back_inserter(indexes), + [](auto const &pair) { return pair.first; }); + auto it = series.iterations.begin(); + auto end = series.iterations.end(); + for (; it != end; ++it) + { + try + { + it->second.open(); + break; + } + catch (error::ReadError const &err) + { + std::cerr << "[StatefulIterator] Cannot read iteration '" + << it->first << "' and will skip it due to read error:\n" + << err.what() << std::endl; + } + } + if (it != end) + { + data.currentStep = + CurrentStep::During_t{0, it->first, std::move(indexes)}; + } + else + { + this->turn_into_end_iterator(TypeOfEndIterator::NoMoreIterationsInStep); + } +} + +void StatefulIterator::deactivateDeadIteration(iteration_index_t index) +{ + auto &data = get(); + switch (data.series.iterationEncoding()) + { + case IterationEncoding::fileBased: { + Parameter param; + data.series.IOHandler()->enqueue( + IOTask(&data.series.iterations[index], std::move(param))); + data.series.IOHandler()->flush({FlushLevel::UserFlush}); + } + break; + case IterationEncoding::variableBased: + case IterationEncoding::groupBased: { + Parameter param; + param.mode = AdvanceMode::ENDSTEP; + data.series.IOHandler()->enqueue( + IOTask(&data.series.iterations[index], std::move(param))); + data.series.IOHandler()->flush({FlushLevel::UserFlush}); + } + break; + } + data.series.iterations.container().erase(index); +} + +StatefulIterator &StatefulIterator::operator++() +{ + return seek({Seek::Next}); +} + +auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator & +{ + std::optional res; + /* + * loopBody() might return an empty option to indicate a skipped iteration. + * Loop until it returns something real for us. + * Note that this is not an infinite loop: + * Upon end of the Series, loopBody() does not return an empty option, + * but the end iterator. + */ + do + { + res = loopBody(seek); + } while (!res.has_value()); + + return **res; +} + +auto StatefulIterator::operator*() -> value_type & +{ + return const_cast( + static_cast(this)->operator*()); +} + +auto StatefulIterator::operator*() const -> value_type const & +{ + auto &data = get(); + if (auto cur = data.currentIteration(); cur.has_value()) + { + + auto iterator = static_cast( + data.series.iterations) + .find(*cur); + return iterator.operator*(); + } + else + { + throw std::runtime_error("No iteration currently active."); + } +} + +auto StatefulIterator::operator--() -> StatefulIterator & +{ + throw error::WrongAPIUsage( + "Global stateful iterator supports no decrement (yet)."); +} +auto StatefulIterator::operator--(int) -> StatefulIterator +{ + throw error::WrongAPIUsage( + "Global stateful iterator supports no post-decrement."); +} +auto StatefulIterator::operator++(int) -> StatefulIterator +{ + throw error::WrongAPIUsage( + "Global stateful iterator supports no post-increment."); +} + +// comparison +auto StatefulIterator::operator-(StatefulIterator const &) const + -> difference_type +{ + throw error::WrongAPIUsage( + "Global stateful iterator supports no relative comparison."); +} +bool StatefulIterator::operator==(StatefulIterator const &other) const +{ + return + // either both iterators are filled + (this->m_data->has_value() && other.m_data->has_value() && + (this->get().currentIteration() == other.get().currentIteration())) || + // or both are empty + (this->is_end() && other.is_end()); +} +auto StatefulIterator::operator<(StatefulIterator const &) const -> bool +{ + throw error::WrongAPIUsage( + "Global stateful iterator supports no relative comparison."); +} + +StatefulIterator StatefulIterator::end() +{ + return StatefulIterator{}; +} + +auto StatefulIterator::is_end() const -> bool +{ + return !m_data || !m_data->has_value() || + std::visit( + auxiliary::overloaded{ + [](CurrentStep::Before_t const &) { return false; }, + [](CurrentStep::During_t const &during) { + return !during.iteration_idx.has_value(); + }, + [](CurrentStep::After_t const &) { return true; }}, + (**m_data).currentStep.as_base()); +} + +auto StatefulIterator::assert_end_iterator() const -> void +{ + if (!is_end()) + { + throw error::Internal("Assertion error: Iterator is no end iterator."); + } +} + +StatefulIterator::operator bool() const +{ + return m_data->has_value(); +} +} // namespace openPMD diff --git a/test/Files_SerialIO/SerialIOTests.hpp b/test/Files_SerialIO/SerialIOTests.hpp new file mode 100644 index 0000000000..541d4dcaf1 --- /dev/null +++ b/test/Files_SerialIO/SerialIOTests.hpp @@ -0,0 +1,10 @@ +#include + +namespace filebased_write_test +{ +auto close_and_reopen_iterations(std::string const &filename) -> void; +} +namespace close_and_reopen_test +{ +auto close_and_reopen_test() -> void; +} diff --git a/test/Files_SerialIO/close_and_reopen_test.cpp b/test/Files_SerialIO/close_and_reopen_test.cpp new file mode 100644 index 0000000000..dcc1969ded --- /dev/null +++ b/test/Files_SerialIO/close_and_reopen_test.cpp @@ -0,0 +1,328 @@ +#include "SerialIOTests.hpp" +#include "openPMD/IO/ADIOS/macros.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/auxiliary/Filesystem.hpp" + +#include + +namespace close_and_reopen_test +{ +using namespace openPMD; +#if openPMD_HAVE_ADIOS2 + +constexpr char const *write_cfg = +#if openPMD_HAS_ADIOS_2_9 + R"(adios2.use_group_table = true + adios2.modifiable_attributes = true)"; +#else + R"(adios2.use_group_table = false + adios2.modifiable_attributes = false)"; +#endif + +template +auto run_test_filebased( + WriteIterations &&writeIterations, std::string const &ext) +{ + std::string filename = + "../samples/close_iteration_reopen/filebased_%T." + ext; + auxiliary::remove_directory("../samples/close_iteration_reopen"); + Series series(filename, Access::CREATE, write_cfg); + { + auto it = writeIterations(series)[0]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + auto B_y = it.meshes["B"]["y"]; + B_y.resetDataset({Datatype::INT, {5}}); + B_y.storeChunk(data, {0}, {5}); + it.close(); + } + + { + auto it = writeIterations(series)[1]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + auto e_position_x = it.particles["e"]["position"]["x"]; + e_position_x.resetDataset({Datatype::INT, {5}}); + e_position_x.storeChunk(data, {0}, {5}); + it.close(); + } + { + auto it = writeIterations(series)[2]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + +#if !openPMD_HAS_ADIOS_2_8 + if (series.backend() != "ADIOS2") + { +#endif + it.open(); + it.setTimeUnitSI(2.0); + it.close(); +#if !openPMD_HAS_ADIOS_2_8 + } +#endif + } + series.close(); + + series = Series(filename, Access::READ_WRITE, write_cfg); + + { + auto it = series.snapshots()[0].open(); + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = series.snapshots()[2].open(); + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + // no guarantee which attribute version we get + REQUIRE((it.timeUnitSI() == 2.0 || it.timeUnitSI() == 1.0)); + } + + { + auto it = series.snapshots()[3].open(); + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + auto e_position_x = it.particles["e"]["position"]["x"]; + e_position_x.resetDataset({Datatype::INT, {5}}); + e_position_x.storeChunk(data, {0}, {5}); + it.close(); + } + series.close(); + + for (auto mode : {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}) + { + Series read(filename, mode); + { + auto it = read.snapshots()[0]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + REQUIRE(read.iterations.size() == 4); + { + auto it = read.snapshots()[1]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = read.snapshots()[3]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = read.snapshots()[2]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + // no guarantee which attribute version we get + REQUIRE((it.timeUnitSI() == 2.0 || it.timeUnitSI() == 1.0)); + } + { + auto it = read.snapshots()[0].open(); + std::vector data(5); + it.meshes["B"]["y"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = read.snapshots()[1].open(); + std::vector data(5); + it.particles["e"]["position"]["x"].loadChunkRaw( + data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = read.snapshots()[3].open(); + std::vector data(5); + it.particles["e"]["position"]["x"].loadChunkRaw( + data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + } +} + +template +auto run_test_groupbased( + WriteIterations &&writeIterations, + std::string const &ext, + std::vector const &readModes) +{ + std::string filename = + "../samples/close_iteration_reopen/groupbased." + ext; + Series series(filename, Access::CREATE, write_cfg); + { + auto it = writeIterations(series)[0]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + auto B_y = it.meshes["B"]["y"]; + B_y.resetDataset({Datatype::INT, {5}}); + B_y.storeChunk(data, {0}, {5}); + it.close(); + } + + { + auto it = writeIterations(series)[1]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + auto E_y = it.meshes["E"]["y"]; + E_y.resetDataset({Datatype::INT, {5}}); + E_y.storeChunk(data, {0}, {5}); + it.close(); + } + { + auto it = writeIterations(series)[2]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + it.setTimeUnitSI(2.0); + it.close(); + } + series.close(); + + for (auto mode : readModes) + { + Series read(filename, mode); + { + auto it = read.snapshots()[0]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + REQUIRE(read.iterations.size() == 3); + { + auto it = read.snapshots()[1]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = read.snapshots()[2]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + // no guarantee which attribute version we get + REQUIRE((it.timeUnitSI() == 2.0 || it.timeUnitSI() == 1.0)); + } + { + auto it = read.snapshots()[0].open(); + std::vector data(5); + it.meshes["B"]["y"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = read.snapshots()[1].open(); + std::vector data(5); + it.meshes["E"]["y"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + } +} + +auto close_and_reopen_test() -> void +{ + run_test_filebased([](Series &s) { return s.iterations; }, "bp"); + run_test_filebased([](Series &s) { return s.writeIterations(); }, "bp"); + run_test_filebased([](Series &s) { return s.snapshots(); }, "bp"); + run_test_filebased( + [](Series &s) { return s.snapshots(SnapshotWorkflow::Synchronous); }, + "bp"); + run_test_filebased( + [](Series &s) { return s.snapshots(SnapshotWorkflow::RandomAccess); }, + "bp"); + run_test_filebased([](Series &s) { return s.snapshots(); }, "json"); +#if openPMD_HAVE_HDF5 + run_test_filebased([](Series &s) { return s.snapshots(); }, "h5"); +#endif + + /* + * This test writes the same attribute with different values over steps, + * triggering a bug in ADIOS2 v2.7. + */ +#if openPMD_HAS_ADIOS_2_8 + run_test_groupbased( + [](Series &s) { return s.iterations; }, + "bp4", + {Access::READ_ONLY, Access::READ_LINEAR}); + // since these write data in a way that distributes one iteration's data + // over multiple steps, only random access read mode makes sense + run_test_groupbased( + [](Series &s) { return s.writeIterations(); }, + "bp4", + {Access::READ_RANDOM_ACCESS}); + run_test_groupbased( + [](Series &s) { return s.snapshots(); }, + "bp4", + {Access::READ_RANDOM_ACCESS}); + // that doesnt matter for json tho + run_test_groupbased( + [](Series &s) { return s.snapshots(); }, + "json", + {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}); +#endif +#if openPMD_HAVE_HDF5 + run_test_groupbased( + [](Series &s) { return s.snapshots(); }, + "h5", + {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}); +#endif + run_test_groupbased( + [](Series &s) { return s.snapshots(); }, + "json", + {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}); +} +#else +auto close_and_reopen_test() -> void +{} +#endif +} // namespace close_and_reopen_test diff --git a/test/Files_SerialIO/filebased_write_test.cpp b/test/Files_SerialIO/filebased_write_test.cpp new file mode 100644 index 0000000000..c7b200b70e --- /dev/null +++ b/test/Files_SerialIO/filebased_write_test.cpp @@ -0,0 +1,116 @@ +#include "SerialIOTests.hpp" +#include "openPMD/IO/Access.hpp" + +namespace filebased_write_test +{ +using namespace openPMD; + +#define OPENPMD_TEST_VERBOSE 0 + +namespace +{ + template + auto write_to_stdout([[maybe_unused]] Args &&...args) -> void + { +#if OPENPMD_TEST_VERBOSE + (std::cout << ... << args); +#endif + } +} // namespace + +auto close_and_reopen_iterations( + const std::string &filename, + openPMD::Access access, + std::string const &json_config, + bool need_to_explitly_open_iterations) -> void +{ + Series list(filename, access, json_config); + + auto test_read = [](Iteration &iteration) { + auto component = iteration.particles["e"]["position"]["x"]; + auto chunk = component.loadChunkVariant(); + iteration.seriesFlush(); + auto num_particles = component.getExtent()[0]; + write_to_stdout("Particles: "); + if (num_particles > 0) + { + std::visit( + [&](auto const &shared_ptr) { + auto it = shared_ptr.get(); + auto end = it + num_particles; + write_to_stdout('[', *it++); + for (; it != end; ++it) + { + write_to_stdout(", ", *it); + } + }, + chunk); + write_to_stdout("]"); + } + else + { + write_to_stdout("[]"); + } + write_to_stdout('\n'); + }; + + for (auto &[idx, iteration] : list.snapshots()) + { + write_to_stdout("Seeing iteration ", idx, '\n'); + if (need_to_explitly_open_iterations) + { + iteration.open(); + } + if (iteration.particles.contains("e")) + { + test_read(iteration); + } + write_to_stdout("Closing iteration ", idx, '\n'); + iteration.close(); + } + write_to_stdout("Trying to read iteration 3 out of line", '\n'); + if (need_to_explitly_open_iterations || access == Access::READ_ONLY) + { + list.snapshots()[3].open(); + } + test_read(list.snapshots()[3]); + + write_to_stdout("----------\nGoing again\n----------", '\n'); + for (auto &[idx, iteration] : list.snapshots()) + { + write_to_stdout("Seeing iteration ", idx, '\n'); + if (need_to_explitly_open_iterations || access == Access::READ_ONLY) + { + iteration.open(); + } + if (iteration.particles.contains("e")) + { + test_read(iteration); + } + } +} + +auto close_and_reopen_iterations(std::string const &filename) -> void +{ + close_and_reopen_iterations( + filename, + Access::READ_LINEAR, + R"({"defer_iteration_parsing":false})", + false); + close_and_reopen_iterations( + filename, + Access::READ_LINEAR, + R"({"defer_iteration_parsing":true})", + false); + close_and_reopen_iterations( + filename, + Access::READ_ONLY, + R"({"defer_iteration_parsing":false})", + false); + close_and_reopen_iterations( + filename, + Access::READ_ONLY, + R"({"defer_iteration_parsing":true})", + true); +} +} // namespace filebased_write_test diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 58dc0b06a2..519a8749b2 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -802,8 +802,8 @@ void close_iteration_test(std::string const &file_ending) it1.close(/* flush = */ true); // illegally access iteration after closing - E_x.storeChunk(data, {mpi_rank, 0}, {1, 4}); - REQUIRE_THROWS(write.flush()); + // E_x.storeChunk(data, {mpi_rank, 0}, {1, 4}); + // REQUIRE_THROWS(write.flush()); } { @@ -817,7 +817,7 @@ void close_iteration_test(std::string const &file_ending) REQUIRE(data[i % 4] == chunk.get()[i]); } auto read_again = E_x_read.loadChunk({0, 0}, {mpi_size, 4}); - REQUIRE_THROWS(read.flush()); + // REQUIRE_THROWS(read.flush()); } chunk_assignment::RankMeta compare; @@ -1829,9 +1829,9 @@ void append_mode( ++counter; } REQUIRE(counter == 8); - // listSeries will not see any iterations since they have already - // been read - helper::listSeries(read); + // @todo this does not work (yet), but should at the end of the + // current project + // helper::listSeries(read); } } #endif diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 325cab4031..41de1bdbcd 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -8,6 +8,8 @@ #define OPENPMD_protected public: #endif +#include "SerialIOTests.hpp" + #include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" @@ -37,6 +39,7 @@ #ifdef __unix__ #include #include +#include #include #include #include @@ -174,6 +177,13 @@ TEST_CASE("char_roundtrip", "[serial]") void write_and_read_many_iterations( std::string const &ext, bool intermittentFlushes) { +#ifdef __unix__ + struct rlimit rlim; + getrlimit(RLIMIT_NOFILE, &rlim); + auto old_soft_limit = rlim.rlim_cur; + rlim.rlim_cur = 512; + setrlimit(RLIMIT_NOFILE, &rlim); +#endif // the idea here is to trigger the maximum allowed number of file handles, // e.g., the upper limit in "ulimit -n" (default: often 1024). Once this // is reached, files should be closed automatically for open iterations @@ -186,6 +196,7 @@ void write_and_read_many_iterations( auxiliary::getEnvNum("OPENPMD_TEST_NFILES_MAX", 1030); std::string filename = "../samples/many_iterations/many_iterations_%T." + ext; + // std::cout << "WRITE " << filename << std::endl; std::vector data(10); std::iota(data.begin(), data.end(), 0.); @@ -211,6 +222,7 @@ void write_and_read_many_iterations( } // ~Series intentionally not yet called + // std::cout << "READ " << filename << std::endl; Series read( filename, Access::READ_ONLY, "{\"defer_iteration_parsing\": true}"); for (auto iteration : read.iterations) @@ -236,6 +248,10 @@ void write_and_read_many_iterations( Series list(filename, Access::READ_ONLY); helper::listSeries(list); +#ifdef __unix__ + rlim.rlim_cur = old_soft_limit; + setrlimit(RLIMIT_NOFILE, &rlim); +#endif } TEST_CASE("write_and_read_many_iterations", "[serial]") @@ -475,10 +491,6 @@ void close_iteration_test(std::string const &file_ending) E_x.resetDataset({Datatype::INT, {2, 2}}); E_x.storeChunk(data, {0, 0}, {2, 2}); it1.close(/* flush = */ true); - - // illegally access iteration after closing - E_x.storeChunk(data, {0, 0}, {2, 2}); - REQUIRE_THROWS(write.flush()); } { @@ -491,8 +503,6 @@ void close_iteration_test(std::string const &file_ending) { REQUIRE(data[i] == chunk.get()[i]); } - auto read_again = E_x_read.loadChunk({0, 0}, {2, 2}); - REQUIRE_THROWS(read.flush()); } { @@ -761,52 +771,9 @@ TEST_CASE("close_and_copy_attributable_test", "[serial]") } #if openPMD_HAVE_ADIOS2 -TEST_CASE("close_iteration_throws_test", "[serial]") +TEST_CASE("close_and_reopen_test", "[serial]") { - /* - * Iterations should not be accessed any more after closing. - * Test that the openPMD API detects that case and throws. - */ - { - Series series("../samples/close_iteration_throws_1.bp", Access::CREATE); - auto it0 = series.iterations[0]; - auto E_x = it0.meshes["E"]["x"]; - E_x.resetDataset({Datatype::INT, {5}}); - std::vector data{0, 1, 2, 3, 4}; - E_x.storeChunk(data, {0}, {5}); - it0.close(); - - auto B_y = it0.meshes["B"]["y"]; - B_y.resetDataset({Datatype::INT, {5}}); - B_y.storeChunk(data, {0}, {5}); - REQUIRE_THROWS(series.flush()); - } - { - Series series("../samples/close_iteration_throws_2.bp", Access::CREATE); - auto it0 = series.iterations[0]; - auto E_x = it0.meshes["E"]["x"]; - E_x.resetDataset({Datatype::INT, {5}}); - std::vector data{0, 1, 2, 3, 4}; - E_x.storeChunk(data, {0}, {5}); - it0.close(); - - auto e_position_x = it0.particles["e"]["position"]["x"]; - e_position_x.resetDataset({Datatype::INT, {5}}); - e_position_x.storeChunk(data, {0}, {5}); - REQUIRE_THROWS(series.flush()); - } - { - Series series("../samples/close_iteration_throws_3.bp", Access::CREATE); - auto it0 = series.iterations[0]; - auto E_x = it0.meshes["E"]["x"]; - E_x.resetDataset({Datatype::INT, {5}}); - std::vector data{0, 1, 2, 3, 4}; - E_x.storeChunk(data, {0}, {5}); - it0.close(); - - it0.setTimeUnitSI(2.0); - REQUIRE_THROWS(series.flush()); - } + close_and_reopen_test::close_and_reopen_test(); } #endif @@ -2266,6 +2233,9 @@ inline void fileBased_write_test(const std::string &backend) close(dirfd); } #endif // defined(__unix__) + + filebased_write_test::close_and_reopen_iterations( + "../samples/subdir/serial_fileBased_write%T." + backend); } TEST_CASE("fileBased_write_test", "[serial]") @@ -5703,18 +5673,24 @@ void adios2_group_table( write.writeIterations()[1].meshes["E"]["x"].makeEmpty(Datatype::FLOAT, 1); write.close(); + size_t counter = 0; + bool saw_iteration_0{false}, saw_iteration_1{false}; + Series read("../samples/group_table.bp", Access::READ_LINEAR, jsonRead); // NOLINTNEXTLINE(performance-for-range-copy) for (auto iteration : read.readIterations()) { + ++counter; switch (iteration.iterationIndex) { case 0: + saw_iteration_0 = true; REQUIRE(iteration.meshes["E"].contains("x")); REQUIRE(iteration.meshes["E"].contains("y")); REQUIRE(iteration.meshes["E"].size() == 2); break; case 1: + saw_iteration_1 = true; if (canDeleteGroups) { REQUIRE(iteration.meshes["E"].contains("x")); @@ -5729,6 +5705,9 @@ void adios2_group_table( break; } } + REQUIRE(counter == 2); + REQUIRE(saw_iteration_0); + REQUIRE(saw_iteration_1); } TEST_CASE("adios2_group_table", "[serial]") @@ -6660,6 +6639,28 @@ void deferred_parsing(std::string const &extension) std::numeric_limits::epsilon()); } } + { + Series series( + basename + "%06T." + extension, + Access::READ_ONLY, + "{\"defer_iteration_parsing\": true}"); + for (auto iteration : series.readIterations()) + { + auto dataset = + iteration.meshes["E"]["x"].loadChunk({0}, {20}); + iteration.close(); + for (size_t i = 0; i < 20; ++i) + { + REQUIRE( + std::abs(dataset.get()[i] - float(i)) <= + std::numeric_limits::epsilon()); + } + if (iteration.iterationIndex == 0) + { + break; + } + } + } { Series series( basename + "%06T." + extension,