From b38a40f64129731ce8c4119f8896e06aa40dc946 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 31 Jan 2023 14:22:47 -0800 Subject: [PATCH] ADIOS2: Fix BTD Resize w/ Empty Ranks For ADIOS2 BP4 and BP5 writes, setting an update of a variable's (openPMD record's) shape is not sufficient to drop all meta-data on disk. In situations where the last write to a BTD stage only adds further particles from a few ranks, we need to pad with zero-block writes so ADIOS2 `Put` gets called. Backend details: - BP4 (as of ADIOS 2.8): last MPI rank's `Put` meta-data wins - BP5 (as of ADIOS 2.8): everyone has to write an empty block --- Source/Diagnostics/WarpXOpenPMD.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 2cee7db218e..1b0fc2d4a62 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -754,7 +754,17 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // Do not call storeChunk() with zero-sized particle tiles: // https://github.com/openPMD/openPMD-api/issues/1147 - if (numParticleOnTile == 0) continue; + // https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290 + // unless we append in ADIOS2: + // https://github.com/ECP-WarpX/WarpX/issues/3389 + // https://github.com/ornladios/ADIOS2/issues/3455 + // BP4 (ADIOS 2.8): last MPI rank's `Put` meta-data wins + // BP5 (ADIOS 2.8): everyone has to write an empty block + bool write_empty_blocks = false; + if (m_Series->backend() == "MPI_ADIOS1" || + m_Series->backend() == "ADIOS2") + write_empty_blocks = isBTD; + if (numParticleOnTile == 0 && !write_empty_blocks) continue; // get position and particle ID from aos // note: this implementation iterates the AoS 4x...