Skip to content

Commit

Permalink
openPMD: ionizationLevel (#1622)
Browse files Browse the repository at this point in the history 
* openPMD: ionizationLevel

Write out the ionizationLevel with openPMD.

* openPMD Record Repetitions: Use Lambdas
  • Loading branch information
@ax3l
ax3l authored Jan 20, 2021
1 parent 1bb6165 commit 1f3191f
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 63 deletions.
21 changes: 16 additions & 5 deletions Source/Diagnostics/WarpXOpenPMD.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,15 @@ private:
* @param[in] currSpecies The openPMD species
* @param[in] write_real_comp The real attribute ids, from WarpX
* @param[in] real_comp_names The real attribute names, from WarpX
* @param[in] write_int_comp The int attribute ids, from WarpX
* @param[in] int_comp_names The int attribute names, from WarpX
* @param[in] np Number of particles
*/
void SetupRealProperties(openPMD::ParticleSpecies& currSpecies,
void SetupSoAProperties (openPMD::ParticleSpecies& currSpecies,
const amrex::Vector<int>& write_real_comp,
const amrex::Vector<std::string>& real_comp_names,
const amrex::Vector<int>& write_int_comp,
const amrex::Vector<std::string>& int_comp_names,
unsigned long long np) const;

/** This function saves the values of the entries for particle properties
Expand All @@ -147,12 +151,16 @@ private:
* @param[in] offset offset to start saving the particle iterator contents
* @param[in] write_real_comp The real attribute ids, from WarpX
* @param[in] real_comp_names The real attribute names, from WarpX
* @param[in] write_int_comp The int attribute ids, from WarpX
* @param[in] int_comp_names The int attribute names, from WarpX
*/
void SaveRealProperty(WarpXParIter& pti, //int, int,
void SaveParticleProperties (WarpXParIter& pti, //int, int,
openPMD::ParticleSpecies& currSpecies,
unsigned long long offset,
const amrex::Vector<int>& write_real_comp,
const amrex::Vector<std::string>& real_comp_names) const;
const amrex::Vector<std::string>& real_comp_names,
const amrex::Vector<int>& write_int_comp,
const amrex::Vector<std::string>& int_comp_names) const;

/** This function saves the plot file
*
Expand Down Expand Up @@ -180,8 +188,11 @@ private:
int m_MPIRank = 0;
int m_MPISize = 1;

int m_NumSoARealAttributes = PIdx::nattribs; //! WarpX' addition particle attributes in SoA
int m_NumAoSRealAttributes = 0; //! WarpX definition: no additional attributes in particle AoS
int m_NumSoARealAttributes = PIdx::nattribs; //! WarpX' additional real particle attributes in SoA
int m_NumAoSRealAttributes = 0; //! WarpX definition: no additional real attributes in particle AoS

//int m_NumSoAIntAttributes = PIdx::nattribs; //! WarpX' additional int particle attributes in SoA
int m_NumAoSIntAttributes = 0; //! WarpX definition: no additional int attributes in particle AoS

bool m_OneFilePerTS = true; //! write in openPMD fileBased manner for individual time steps
std::string m_OpenPMDFileType = "bp"; //! MPI-parallel openPMD backend: bp or h5
Expand Down
160 changes: 102 additions & 58 deletions Source/Diagnostics/WarpXOpenPMD.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,9 @@ namespace detail
{
std::string record_name = fullName;
std::string component_name = openPMD::RecordComponent::SCALAR;
std::size_t startComp = fullName.find_last_of("_");

// we use "_" as separator in names to group vector records
std::size_t startComp = fullName.find_last_of("_");
if( startComp != std::string::npos ) { // non-scalar
record_name = fullName.substr(0, startComp);
component_name = fullName.substr(startComp + 1u);
Expand Down Expand Up @@ -327,7 +328,7 @@ WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector<ParticleDiag>& part
real_names.push_back("theta");
#endif
if(pc->DoFieldIonization()){
int_names.push_back("ionization_level");
int_names.push_back("ionizationLevel");
// int_flags specifies, for each integer attribs, whether it is
// dumped as particle record in a plotfile. So far, ionization_level is the only
// integer attribs, and it is automatically dumped as particle record
Expand Down Expand Up @@ -364,10 +365,6 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc,
const amrex::Vector<std::string>& int_comp_names) const
{
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_Series != nullptr, "openPMD: series must be initialized");
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(write_int_comp.size() == 0u,
"openPMD: Particle integer components not implemented!");
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(int_comp_names.size() == 0u,
"openPMD: Particle integer components not implemented!");

WarpXParticleCounter counter(pc);

Expand Down Expand Up @@ -413,7 +410,7 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc,
// define positions & offsets
//
SetupPos(pc, currSpecies, counter.GetTotalNumParticles());
SetupRealProperties(currSpecies, write_real_comp, real_comp_names, counter.GetTotalNumParticles());
SetupSoAProperties(currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, counter.GetTotalNumParticles());

// open files from all processors, in case some will not contribute below
m_Series->flush();
Expand Down Expand Up @@ -494,10 +491,11 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc,
currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64});
}
// save "extra" particle properties in AoS and SoA
SaveRealProperty(pti,
SaveParticleProperties(pti,
currSpecies,
offset,
write_real_comp, real_comp_names);
write_real_comp, real_comp_names,
write_int_comp, int_comp_names);

offset += numParticleOnTile64;
}
Expand All @@ -506,58 +504,92 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc,
}

void
WarpXOpenPMDPlot::SetupRealProperties(openPMD::ParticleSpecies& currSpecies,
WarpXOpenPMDPlot::SetupSoAProperties (openPMD::ParticleSpecies& currSpecies,
const amrex::Vector<int>& write_real_comp,
const amrex::Vector<std::string>& real_comp_names,
const amrex::Vector<int>& write_int_comp,
const amrex::Vector<std::string>& int_comp_names,
unsigned long long np) const
{
auto particlesLineup = openPMD::Dataset(openPMD::determineDatatype<amrex::ParticleReal>(), {np});

//
// the beam/input3d showed write_real_comp.size() = 16 while only 10 real comp names
// so using the min to be safe.
//
auto counter = std::min(write_real_comp.size(), real_comp_names.size());
for (int i = 0; i < counter; ++i)
if (write_real_comp[i]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(real_comp_names[i]);

auto particleVarComp = currSpecies[record_name][component_name];
particleVarComp.resetDataset(particlesLineup);
auto dtype_real = openPMD::Dataset(openPMD::determineDatatype<amrex::ParticleReal>(), {np});
auto dtype_int = openPMD::Dataset(openPMD::determineDatatype<int>(), {np});

//
// the beam/input3d showed write_real_comp.size() = 16 while only 10 real comp names
// so using the min to be safe.
//
auto const getComponentRecord = [&currSpecies](std::string const comp_name) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(comp_name);
return currSpecies[record_name][component_name];
};
auto const real_counter = std::min(write_real_comp.size(), real_comp_names.size());
for (int i = 0; i < real_counter; ++i) {
if (write_real_comp[i]) {
getComponentRecord(real_comp_names[i]).resetDataset(dtype_real);
}
}
auto const int_counter = std::min(write_int_comp.size(), int_comp_names.size());
for (int i = 0; i < int_counter; ++i) {
if (write_int_comp[i]) {
getComponentRecord(int_comp_names[i]).resetDataset(dtype_int);
}
}

std::set< std::string > addedRecords; // add meta-data per record only once
for (auto idx=0; idx<m_NumSoARealAttributes; idx++) {
auto ii = m_NumAoSRealAttributes + idx;
if (write_real_comp[ii]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(real_comp_names[ii]);
auto currRecord = currSpecies[record_name];

// meta data for ED-PIC extension
bool newRecord = false;
std::tie(std::ignore, newRecord) = addedRecords.insert(record_name);
if( newRecord ) {
currRecord.setUnitDimension( detail::getUnitDimension(record_name) );
currRecord.setAttribute( "macroWeighted", 0u );
if( record_name == "momentum" )
currRecord.setAttribute( "weightingPower", 1.0 );
else
currRecord.setAttribute( "weightingPower", 0.0 );
}
std::set< std::string > addedRecords; // add meta-data per record only once
for (auto idx=0; idx<m_NumSoARealAttributes; idx++) {
auto ii = m_NumAoSRealAttributes + idx; // jump over AoS names
if (write_real_comp[ii]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(real_comp_names[ii]);
auto currRecord = currSpecies[record_name];

// meta data for ED-PIC extension
bool newRecord = false;
std::tie(std::ignore, newRecord) = addedRecords.insert(record_name);
if( newRecord ) {
currRecord.setUnitDimension( detail::getUnitDimension(record_name) );
currRecord.setAttribute( "macroWeighted", 0u );
if( record_name == "momentum" )
currRecord.setAttribute( "weightingPower", 1.0 );
else
currRecord.setAttribute( "weightingPower", 0.0 );
}
}
}
for (auto idx=0; idx<int_counter; idx++) {
auto ii = m_NumAoSIntAttributes + idx; // jump over AoS names
if (write_int_comp[ii]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(int_comp_names[ii]);
auto currRecord = currSpecies[record_name];

// meta data for ED-PIC extension
bool newRecord = false;
std::tie(std::ignore, newRecord) = addedRecords.insert(record_name);
if( newRecord ) {
currRecord.setUnitDimension( detail::getUnitDimension(record_name) );
currRecord.setAttribute( "macroWeighted", 0u );
if( record_name == "momentum" )
currRecord.setAttribute( "weightingPower", 1.0 );
else
currRecord.setAttribute( "weightingPower", 0.0 );
}
}
}
}
}

void
WarpXOpenPMDPlot::SaveRealProperty(WarpXParIter& pti,
WarpXOpenPMDPlot::SaveParticleProperties(WarpXParIter& pti,
openPMD::ParticleSpecies& currSpecies,
unsigned long long const offset,
amrex::Vector<int> const& write_real_comp,
amrex::Vector<std::string> const& real_comp_names) const
amrex::Vector<std::string> const& real_comp_names,
amrex::Vector<int> const& write_int_comp,
amrex::Vector<std::string> const& int_comp_names) const

{
int numOutputReal = 0;
Expand All @@ -572,7 +604,7 @@ WarpXOpenPMDPlot::SaveRealProperty(WarpXParIter& pti,
auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile
auto const& soa = pti.GetStructOfArrays();

// properties are saved separately
// first we concatinate the AoS into contiguous arrays
{
for( auto idx=0; idx<m_NumAoSRealAttributes; idx++ ) {
if( write_real_comp[idx] ) {
Expand All @@ -596,25 +628,37 @@ WarpXOpenPMDPlot::SaveRealProperty(WarpXParIter& pti,
}
}

auto const getComponentRecord = [&currSpecies](std::string const comp_name) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(comp_name);
return currSpecies[record_name][component_name];
};

// here we the save the SoA properties (real)
{
for (auto idx=0; idx<m_NumSoARealAttributes; idx++) {
auto ii = m_NumAoSRealAttributes + idx;
if (write_real_comp[ii]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(real_comp_names[ii]);
auto& currRecord = currSpecies[record_name];
auto& currRecordComp = currRecord[component_name];

currRecordComp.storeChunk(openPMD::shareRaw(soa.GetRealData(idx)),
{offset}, {numParticleOnTile64});
getComponentRecord(real_comp_names[ii]).storeChunk(openPMD::shareRaw(soa.GetRealData(idx)),
{offset}, {numParticleOnTile64});
}
}
}
// and now SoA int properties
{
auto const int_counter = std::min(write_int_comp.size(), int_comp_names.size());
for (auto idx=0; idx<int_counter; idx++) {
auto ii = m_NumAoSIntAttributes + idx; // jump over AoS names
if (write_int_comp[ii]) {
getComponentRecord(int_comp_names[ii]).storeChunk(openPMD::shareRaw(soa.GetIntData(idx)),
{offset}, {numParticleOnTile64});
}
}
}
}



void
WarpXOpenPMDPlot::SetupPos(WarpXParticleContainer* pc,
openPMD::ParticleSpecies& currSpecies,
Expand Down

0 comments on commit 1f3191f

Please sign in to comment.