Define: _OPENMP ->

Define: _OPENMP -> AMREX_USE_OMP Replace the define check of `_OPENMP` with the explicit backend control of `AMREX_USE_OMP` for parallel constructs. Doing so avoids that we accidentially turn on OpenMP, e.g. if a dependency pulls it in for linear algebra, I/O, etc. This can led to confusion if the user explicitly requested a serial build. Also, we might want to use OpenMP functionality here and there for auxiliary functions w/o having to use the AMReX OpenMP backend, i.e. because we compile for GPUs.
ECP-WarpX · Nov 18, 2020 · 4599ebb · 4599ebb
1 parent 3006545
commit 4599ebb
Show file tree

Hide file tree

Showing 42 changed files with 98 additions and 98 deletions.
diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp
@@ -15,7 +15,7 @@
 #include <AMReX_Print.H>
 #include <AMReX_VisMF.H>
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #   include <omp.h>
 #endif
 
@@ -399,7 +399,7 @@ MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt)
 
     dt_B = dt;
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel
 #endif
     for (MFIter mfi(*this); mfi.isValid(); ++mfi)
@@ -415,7 +415,7 @@ MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt)
 
     dt_E = dt;
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel
 #endif
     for (MFIter mfi(*this); mfi.isValid(); ++mfi)
@@ -899,7 +899,7 @@ PML::Exchange (MultiFab& pml, MultiFab& reg, const Geometry& geom,
         if (ngr.max() > 0) {
             MultiFab::Copy(tmpregmf, reg, 0, 0, 1, ngr);
             tmpregmf.ParallelCopy(totpmlmf, 0, 0, 1, IntVect(0), ngr, period);
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(reg); mfi.isValid(); ++mfi)

diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp
@@ -51,7 +51,7 @@ WarpX::DampPML (int lev, PatchType patch_type)
         const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp()
                                                               : pml[lev]->GetMultiSigmaBox_cp();
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
         for ( MFIter mfi(*pml_E[0], TilingIfNotGPU()); mfi.isValid(); ++mfi )
@@ -172,7 +172,7 @@ WarpX::DampJPML (int lev, PatchType patch_type)
         const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp()
                                                             : pml[lev]->GetMultiSigmaBox_cp();
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
         for ( MFIter mfi(*pml_j[0], TilingIfNotGPU()); mfi.isValid(); ++mfi )

diff --git a/Source/Diagnostics/BackTransformedDiagnostic.cpp b/Source/Diagnostics/BackTransformedDiagnostic.cpp
@@ -477,7 +477,7 @@ LorentzTransformZ(MultiFab& data, Real gamma_boost, Real beta_boost)
 {
     // Loop over tiles/boxes and in-place convert each slice from boosted
     // frame to back-transformed lab frame.
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(data, TilingIfNotGPU()); mfi.isValid(); ++mfi) {

diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp
@@ -152,7 +152,7 @@ void
 BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamma_boost,
                                          amrex::Real beta_boost) const
 {
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (amrex::MFIter mfi(data, TilingIfNotGPU()); mfi.isValid(); ++mfi) {

diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.cpp
@@ -22,7 +22,7 @@ PartPerGridFunctor::operator()(amrex::MultiFab& mf_dst, const int dcomp, const i
     // Temporary MultiFab containing number of particles per grid.
     // (stored as constant for all cells in each grid)
     amrex::MultiFab ppg_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng);
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel
 #endif
     for (amrex::MFIter mfi(ppg_mf); mfi.isValid(); ++mfi) {

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp
@@ -78,7 +78,7 @@ FlushFormatPlotfile::WriteJobInfo(const std::string& dir) const
         jobInfoFile << PrettyLine;
 
         jobInfoFile << "number of MPI processes: " << ParallelDescriptor::NProcs() << "\n";
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
         jobInfoFile << "number of threads:       " << omp_get_max_threads() << "\n";
 #endif
 

diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp
@@ -125,7 +125,7 @@ PhysicalParticleContainer::ConvertUnits(ConvertDirection convert_direction)
 
     const int nLevels = finestLevel();
     for (int lev=0; lev<=nLevels; lev++){
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
         for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti)

diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp
@@ -172,7 +172,7 @@ void FieldMaximum::ComputeDiags (int step)
 #endif
 
         // MFIter loop to interpolate fields to cell center and get maximum values
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
         for ( MFIter mfi(Ex, TilingIfNotGPU()); mfi.isValid(); ++mfi )

diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp
@@ -161,7 +161,7 @@ WarpX::computeE (amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3> >
 
         const Real* dx = Geom(lev).CellSize();
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #    pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
         for ( MFIter mfi(*phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
@@ -263,7 +263,7 @@ WarpX::computeB (amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3> >
 
         const Real* dx = Geom(lev).CellSize();
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #    pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
         for ( MFIter mfi(*phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp
@@ -63,7 +63,7 @@ void FiniteDifferenceSolver::ComputeDivECartesian (
     amrex::MultiFab& divEfield ) {
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(divEfield, TilingIfNotGPU()); mfi.isValid(); ++mfi ) {
@@ -109,7 +109,7 @@ void FiniteDifferenceSolver::ComputeDivECylindrical (
     amrex::MultiFab& divEfield ) {
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(divEfield, TilingIfNotGPU()); mfi.isValid(); ++mfi ) {

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp
@@ -63,7 +63,7 @@ void FiniteDifferenceSolver::EvolveBCartesian (
     amrex::Real const dt ) {
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Bfield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) {
@@ -122,7 +122,7 @@ void FiniteDifferenceSolver::EvolveBCylindrical (
     amrex::Real const dt ) {
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Bfield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) {

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp
@@ -62,7 +62,7 @@ void FiniteDifferenceSolver::EvolveBPMLCartesian (
     amrex::Real const dt ) {
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Bfield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) {

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp
@@ -71,7 +71,7 @@ void FiniteDifferenceSolver::EvolveECartesian (
     Real constexpr c2 = PhysConst::c * PhysConst::c;
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Efield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) {
@@ -165,7 +165,7 @@ void FiniteDifferenceSolver::EvolveECylindrical (
     amrex::Real const dt ) {
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Efield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) {

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp
@@ -76,7 +76,7 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian (
     Real constexpr c2 = PhysConst::c * PhysConst::c;
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Efield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) {

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp
@@ -69,7 +69,7 @@ void FiniteDifferenceSolver::EvolveFCartesian (
     amrex::Real const dt ) {
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Ffield, TilingIfNotGPU()); mfi.isValid(); ++mfi ) {
@@ -122,7 +122,7 @@ void FiniteDifferenceSolver::EvolveFCylindrical (
     amrex::Real const dt ) {
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Ffield, TilingIfNotGPU()); mfi.isValid(); ++mfi ) {

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp
@@ -62,7 +62,7 @@ void FiniteDifferenceSolver::EvolveFPMLCartesian (
     amrex::Real const dt ) {
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Ffield, TilingIfNotGPU()); mfi.isValid(); ++mfi ) {

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp
@@ -94,7 +94,7 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian (
 
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Efield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) {

diff --git a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp
@@ -89,7 +89,7 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, Real a_dt)
     amrex::LayoutData<amrex::Real>* cost = WarpX::getCosts(lev);
 
     // Loop through the grids, and over the tiles within each grid
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
     for ( MFIter mfi(*Bx, TilingIfNotGPU()); mfi.isValid(); ++mfi )

diff --git a/Source/Filter/BilinearFilter.cpp b/Source/Filter/BilinearFilter.cpp
@@ -8,7 +8,7 @@
 #include "BilinearFilter.H"
 #include "WarpX.H"
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #   include <omp.h>
 #endif
 

diff --git a/Source/Filter/Filter.cpp b/Source/Filter/Filter.cpp
@@ -8,7 +8,7 @@
 #include "Filter.H"
 #include "WarpX.H"
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #   include <omp.h>
 #endif
 
@@ -168,7 +168,7 @@ Filter::ApplyStencil (amrex::MultiFab& dstmf, const amrex::MultiFab& srcmf, int
 {
     WARPX_PROFILE("Filter::ApplyStencil(MultiFab)");
     ncomp = std::min(ncomp, srcmf.nComp());
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel
 #endif
     {

diff --git a/Source/Filter/NCIGodfreyFilter.cpp b/Source/Filter/NCIGodfreyFilter.cpp
@@ -8,7 +8,7 @@
 #include "Utils/NCIGodfreyTables.H"
 #include "WarpX.H"
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #   include <omp.h>
 #endif
 

diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp
@@ -428,11 +428,11 @@ LaserParticleContainer::Evolve (int lev,
 
     amrex::LayoutData<amrex::Real>* cost = WarpX::getCosts(lev);
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel
 #endif
     {
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
         int const thread_num = omp_get_thread_num();
 #else
         int const thread_num = 0;

diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp
@@ -71,7 +71,7 @@ void
 WarpX::UpdateAuxilaryDataStagToNodal ()
 {
     // For level 0, we only need to do the average.
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(*Bfield_aux[0][0]); mfi.isValid(); ++mfi)
@@ -130,7 +130,7 @@ WarpX::UpdateAuxilaryDataStagToNodal ()
                 Btmp[i]->ParallelCopy(*Bfield_aux[lev-1][i], 0, 0, 1, ng, ng, cperiod);
             }
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(*Bfield_aux[lev][0]); mfi.isValid(); ++mfi)
@@ -180,7 +180,7 @@ WarpX::UpdateAuxilaryDataStagToNodal ()
                 Etmp[i]->ParallelCopy(*Efield_aux[lev-1][i], 0, 0, 1, ng, ng, cperiod);
             }
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi)
@@ -244,7 +244,7 @@ WarpX::UpdateAuxilaryDataSameType ()
             const int refinement_ratio = refRatio(lev-1)[0];
             AMREX_ALWAYS_ASSERT(refinement_ratio == 2);
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(*Bfield_aux[lev][0]); mfi.isValid(); ++mfi)
@@ -296,7 +296,7 @@ WarpX::UpdateAuxilaryDataSameType ()
             MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, Efield_cp[lev][1]->nComp(), ng);
             MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, Efield_cp[lev][2]->nComp(), ng);
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi)

diff --git a/Source/Parser/GpuParser.H b/Source/Parser/GpuParser.H
@@ -52,7 +52,7 @@ public:
 
 #else
 // WarpX compiled for CPU
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
         int tid = omp_get_thread_num();
 #else
         int tid = 0;
@@ -100,28 +100,28 @@ GpuParser<N>::GpuParser (WarpXParser const& wp)
 
 #else // not defined AMREX_USE_GPU
 
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
     nthreads = omp_get_max_threads();
-#else // _OPENMP
+#else // AMREX_USE_OMP
     nthreads = 1;
-#endif // _OPENMP
+#endif // AMREX_USE_OMP
 
     m_parser = ::new struct wp_parser*[nthreads];
     m_var = ::new amrex::GpuArray<amrex::Real,N>[nthreads];
 
     for (int tid = 0; tid < nthreads; ++tid)
     {
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
         m_parser[tid] = wp_parser_dup(wp.m_parser[tid]);
         for (int i = 0; i < N; ++i) {
             wp_parser_regvar(m_parser[tid], wp.m_varnames[tid][i].c_str(), &(m_var[tid][i]));
         }
-#else // _OPENMP
+#else // AMREX_USE_OMP
         m_parser[tid] = wp_parser_dup(wp.m_parser);
         for (int i = 0; i < N; ++i) {
             wp_parser_regvar(m_parser[tid], wp.m_varnames[i].c_str(), &(m_var[tid][i]));
         }
-#endif // _OPENMP
+#endif // AMREX_USE_OMP
     }
 
 #endif // AMREX_USE_GPU