diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index a37a7e7abed..e7ce1307fd2 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -133,11 +133,15 @@ private: * @param[in] currSpecies The openPMD species * @param[in] write_real_comp The real attribute ids, from WarpX * @param[in] real_comp_names The real attribute names, from WarpX + * @param[in] write_int_comp The int attribute ids, from WarpX + * @param[in] int_comp_names The int attribute names, from WarpX * @param[in] np Number of particles */ - void SetupRealProperties(openPMD::ParticleSpecies& currSpecies, + void SetupSoAProperties (openPMD::ParticleSpecies& currSpecies, const amrex::Vector& write_real_comp, const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, unsigned long long np) const; /** This function saves the values of the entries for particle properties @@ -147,12 +151,16 @@ private: * @param[in] offset offset to start saving the particle iterator contents * @param[in] write_real_comp The real attribute ids, from WarpX * @param[in] real_comp_names The real attribute names, from WarpX + * @param[in] write_int_comp The int attribute ids, from WarpX + * @param[in] int_comp_names The int attribute names, from WarpX */ - void SaveRealProperty(WarpXParIter& pti, //int, int, + void SaveParticleProperties (WarpXParIter& pti, //int, int, openPMD::ParticleSpecies& currSpecies, unsigned long long offset, const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names) const; + const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names) const; /** This function saves the plot file * @@ -180,8 +188,11 @@ private: int m_MPIRank = 0; int m_MPISize = 1; - int m_NumSoARealAttributes = PIdx::nattribs; //! WarpX' addition particle attributes in SoA - int m_NumAoSRealAttributes = 0; //! WarpX definition: no additional attributes in particle AoS + int m_NumSoARealAttributes = PIdx::nattribs; //! WarpX' additional real particle attributes in SoA + int m_NumAoSRealAttributes = 0; //! WarpX definition: no additional real attributes in particle AoS + + //int m_NumSoAIntAttributes = PIdx::nattribs; //! WarpX' additional int particle attributes in SoA + int m_NumAoSIntAttributes = 0; //! WarpX definition: no additional int attributes in particle AoS bool m_OneFilePerTS = true; //! write in openPMD fileBased manner for individual time steps std::string m_OpenPMDFileType = "bp"; //! MPI-parallel openPMD backend: bp or h5 diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index a82d5836a81..3d2bdbf5653 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -34,8 +34,9 @@ namespace detail { std::string record_name = fullName; std::string component_name = openPMD::RecordComponent::SCALAR; - std::size_t startComp = fullName.find_last_of("_"); + // we use "_" as separator in names to group vector records + std::size_t startComp = fullName.find_last_of("_"); if( startComp != std::string::npos ) { // non-scalar record_name = fullName.substr(0, startComp); component_name = fullName.substr(startComp + 1u); @@ -327,7 +328,7 @@ WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector& part real_names.push_back("theta"); #endif if(pc->DoFieldIonization()){ - int_names.push_back("ionization_level"); + int_names.push_back("ionizationLevel"); // int_flags specifies, for each integer attribs, whether it is // dumped as particle record in a plotfile. So far, ionization_level is the only // integer attribs, and it is automatically dumped as particle record @@ -364,10 +365,6 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc, const amrex::Vector& int_comp_names) const { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_Series != nullptr, "openPMD: series must be initialized"); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(write_int_comp.size() == 0u, - "openPMD: Particle integer components not implemented!"); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(int_comp_names.size() == 0u, - "openPMD: Particle integer components not implemented!"); WarpXParticleCounter counter(pc); @@ -413,7 +410,7 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc, // define positions & offsets // SetupPos(pc, currSpecies, counter.GetTotalNumParticles()); - SetupRealProperties(currSpecies, write_real_comp, real_comp_names, counter.GetTotalNumParticles()); + SetupSoAProperties(currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, counter.GetTotalNumParticles()); // open files from all processors, in case some will not contribute below m_Series->flush(); @@ -494,10 +491,11 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc, currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64}); } // save "extra" particle properties in AoS and SoA - SaveRealProperty(pti, + SaveParticleProperties(pti, currSpecies, offset, - write_real_comp, real_comp_names); + write_real_comp, real_comp_names, + write_int_comp, int_comp_names); offset += numParticleOnTile64; } @@ -506,58 +504,92 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc, } void -WarpXOpenPMDPlot::SetupRealProperties(openPMD::ParticleSpecies& currSpecies, +WarpXOpenPMDPlot::SetupSoAProperties (openPMD::ParticleSpecies& currSpecies, const amrex::Vector& write_real_comp, const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, unsigned long long np) const { - auto particlesLineup = openPMD::Dataset(openPMD::determineDatatype(), {np}); - - // - // the beam/input3d showed write_real_comp.size() = 16 while only 10 real comp names - // so using the min to be safe. - // - auto counter = std::min(write_real_comp.size(), real_comp_names.size()); - for (int i = 0; i < counter; ++i) - if (write_real_comp[i]) { - // handle scalar and non-scalar records by name - std::string record_name, component_name; - std::tie(record_name, component_name) = detail::name2openPMD(real_comp_names[i]); - - auto particleVarComp = currSpecies[record_name][component_name]; - particleVarComp.resetDataset(particlesLineup); + auto dtype_real = openPMD::Dataset(openPMD::determineDatatype(), {np}); + auto dtype_int = openPMD::Dataset(openPMD::determineDatatype(), {np}); + + // + // the beam/input3d showed write_real_comp.size() = 16 while only 10 real comp names + // so using the min to be safe. + // + auto const getComponentRecord = [&currSpecies](std::string const comp_name) { + // handle scalar and non-scalar records by name + std::string record_name, component_name; + std::tie(record_name, component_name) = detail::name2openPMD(comp_name); + return currSpecies[record_name][component_name]; + }; + auto const real_counter = std::min(write_real_comp.size(), real_comp_names.size()); + for (int i = 0; i < real_counter; ++i) { + if (write_real_comp[i]) { + getComponentRecord(real_comp_names[i]).resetDataset(dtype_real); + } + } + auto const int_counter = std::min(write_int_comp.size(), int_comp_names.size()); + for (int i = 0; i < int_counter; ++i) { + if (write_int_comp[i]) { + getComponentRecord(int_comp_names[i]).resetDataset(dtype_int); + } } - std::set< std::string > addedRecords; // add meta-data per record only once - for (auto idx=0; idx addedRecords; // add meta-data per record only once + for (auto idx=0; idx const& write_real_comp, - amrex::Vector const& real_comp_names) const + amrex::Vector const& real_comp_names, + amrex::Vector const& write_int_comp, + amrex::Vector const& int_comp_names) const { int numOutputReal = 0; @@ -572,7 +604,7 @@ WarpXOpenPMDPlot::SaveRealProperty(WarpXParIter& pti, auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile auto const& soa = pti.GetStructOfArrays(); - // properties are saved separately + // first we concatinate the AoS into contiguous arrays { for( auto idx=0; idx