Skip to content

Commit

Permalink
Fix optional component communication for pure SoA particles (#4333)
Browse files Browse the repository at this point in the history 
If using the legacy layout, the `AMREX_SPACEDIM` position components are
counted separately. But if using pure SoA, the position components are
assumed to be part of `NArrayReal` - hence the
`h_redistribute_real_comp` vector was the wrong size.

The proposed changes:
- [x] fix a bug or incorrect behavior in AMReX
- [ ] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX
users
- [ ] include documentation in the code and/or rst files, if appropriate
  • Loading branch information
@atmyers
atmyers authored Feb 13, 2025
1 parent 240e2a8 commit 275f55f
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 20 deletions.
20 changes: 5 additions & 15 deletions Src/Particle/AMReX_ParticleContainer.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,9 +199,7 @@ public:
//! of a level hierarchy. Must be properly initialized later.
ParticleContainer_impl ()
:
ParticleContainerBase(),
h_redistribute_real_comp(AMREX_SPACEDIM + NStructReal + NArrayReal, true),
h_redistribute_int_comp(2 + NStructInt + NArrayInt, true)
ParticleContainerBase()
{
Initialize ();
}
Expand All @@ -215,9 +213,7 @@ public:
//!
ParticleContainer_impl (ParGDBBase* gdb)
:
ParticleContainerBase(gdb),
h_redistribute_real_comp(AMREX_SPACEDIM + NStructReal + NArrayReal, true),
h_redistribute_int_comp(2 + NStructInt + NArrayInt, true)
ParticleContainerBase(gdb)
{
Initialize ();
ParticleContainer_impl::reserveData();
Expand All @@ -235,9 +231,7 @@ public:
const DistributionMapping & dmap,
const BoxArray & ba)
:
ParticleContainerBase(geom, dmap, ba),
h_redistribute_real_comp(AMREX_SPACEDIM + NStructReal + NArrayReal, true),
h_redistribute_int_comp(2 + NStructInt + NArrayInt, true)
ParticleContainerBase(geom, dmap, ba)
{
Initialize ();
ParticleContainer_impl::reserveData();
Expand All @@ -258,9 +252,7 @@ public:
const Vector<BoxArray> & ba,
const Vector<int> & rr)
:
ParticleContainerBase(geom, dmap, ba, rr),
h_redistribute_real_comp(AMREX_SPACEDIM + NStructReal + NArrayReal, true),
h_redistribute_int_comp(2 + NStructInt + NArrayInt, true)
ParticleContainerBase(geom, dmap, ba, rr)
{
Initialize ();
ParticleContainer_impl::reserveData();
Expand All @@ -280,9 +272,7 @@ public:
const Vector<BoxArray> & ba,
const Vector<IntVect> & rr)
:
ParticleContainerBase(geom, dmap, ba, rr),
h_redistribute_real_comp(AMREX_SPACEDIM + NStructReal + NArrayReal, true),
h_redistribute_int_comp(2 + NStructInt + NArrayInt, true)
ParticleContainerBase(geom, dmap, ba, rr)
{
Initialize ();
ParticleContainer_impl::reserveData();
Expand Down
34 changes: 29 additions & 5 deletions Src/Particle/AMReX_ParticleContainerI.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ void
ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor>::SetParticleSize ()
{
num_real_comm_comps = 0;
int comm_comps_start = AMREX_SPACEDIM + NStructReal;
int comm_comps_start = 0;
if constexpr (!ParticleType::is_soa_particle) {
comm_comps_start += AMREX_SPACEDIM + NStructReal;
}
for (int i = comm_comps_start; i < comm_comps_start + NumRealComps(); ++i) {
if (h_redistribute_real_comp[i]) {++num_real_comm_comps;}
}
Expand Down Expand Up @@ -45,6 +48,15 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
usePrePost = false;
doUnlink = true;

// by default communicate all components
if constexpr (ParticleType::is_soa_particle)
{
h_redistribute_real_comp.resize(NArrayReal, true);
} else {
h_redistribute_real_comp.resize(AMREX_SPACEDIM + NStructReal + NArrayReal, true);
}
h_redistribute_int_comp.resize(2 + NStructInt + NArrayInt, true);

SetParticleSize();

// add default names for SoA Real and Int compile-time arguments
Expand Down Expand Up @@ -1742,7 +1754,10 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
particles_to_send.resize(new_size);
std::memcpy(&particles_to_send[old_size], &p, particle_size);
char* dst = &particles_to_send[old_size] + particle_size;
int array_comp_start = AMREX_SPACEDIM + NStructReal;
int array_comp_start = 0;
if constexpr (!ParticleType::is_soa_particle) {
array_comp_start = AMREX_SPACEDIM + NStructReal;
}
for (int comp = 0; comp < NumRealComps(); comp++) {
if (h_redistribute_real_comp[array_comp_start + comp]) {
std::memcpy(dst, &soa.GetRealData(comp)[pindex], sizeof(ParticleReal));
Expand Down Expand Up @@ -1866,7 +1881,10 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
std::memcpy(dst, &soa.GetIdCPUData()[pindex], sizeof(uint64_t));
dst += sizeof(uint64_t);
}
int array_comp_start = AMREX_SPACEDIM + NStructReal;
int array_comp_start = 0;
if constexpr (!ParticleType::is_soa_particle) {
array_comp_start = AMREX_SPACEDIM + NStructReal;
}
for (int comp = 0; comp < NumRealComps(); comp++) {
if (h_redistribute_real_comp[array_comp_start + comp]) {
std::memcpy(dst, &soa.GetRealData(comp)[pindex], sizeof(ParticleReal));
Expand Down Expand Up @@ -2267,7 +2285,10 @@ RedistributeMPI (std::map<int, Vector<char> >& not_ours,
ptile.push_back(p);
}

int array_comp_start = AMREX_SPACEDIM + NStructReal;
int array_comp_start = 0;
if constexpr (!ParticleType::is_soa_particle) {
array_comp_start = AMREX_SPACEDIM + NStructReal;
}
for (int comp = 0; comp < NumRealComps(); ++comp) {
if (h_redistribute_real_comp[array_comp_start + comp]) {
ParticleReal rdata;
Expand Down Expand Up @@ -2345,7 +2366,10 @@ RedistributeMPI (std::map<int, Vector<char> >& not_ours,
host_int_attribs[lev][ind].resize(NumIntComps());

// add the real...
int array_comp_start = AMREX_SPACEDIM + NStructReal;
int array_comp_start = 0;
if constexpr (!ParticleType::is_soa_particle) {
array_comp_start = AMREX_SPACEDIM + NStructReal;
}
for (int comp = 0; comp < NumRealComps(); ++comp) {
if (h_redistribute_real_comp[array_comp_start + comp]) {
Real rdata;
Expand Down

0 comments on commit 275f55f

Please sign in to comment.