Skip to content

Commit

Permalink
turn on lazy QueueReduction for the timing reductions (#2926)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zingale
zingale authored Jul 28, 2024
1 parent 3fd2d9c commit bb30ff4
Show file tree
Hide file tree
Showing 13 changed files with 80 additions and 50 deletions.
3 changes: 3 additions & 0 deletions Exec/Make.Castro
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ endif
# Require C++17
CXXSTD := c++17

# Use Lazy QueueReduction for the timing outputs
LAZY := TRUE

# default integrator
INTEGRATOR_DIR ?= VODE

Expand Down
17 changes: 10 additions & 7 deletions Source/diffusion/Castro_diffusion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,16 @@ Castro::construct_old_diff_source(MultiFab& source, MultiFab& state_in, Real tim

if (verbose > 1)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;

const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;
#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::construct_old_diff_source() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::construct_old_diff_source() time = " << run_time
<< " on level " << llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down Expand Up @@ -59,15 +60,17 @@ Castro::construct_new_diff_source(MultiFab& source, MultiFab& state_old, MultiFa

if (verbose > 1)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::construct_new_diff_source() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::construct_new_diff_source() time = " << run_time
<< " on level " << llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down
6 changes: 4 additions & 2 deletions Source/driver/Castro.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3048,14 +3048,16 @@ Castro::reflux (int crse_level, int fine_level, bool in_post_timestep)
if (verbose)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real end = ParallelDescriptor::second() - strt;
amrex::Real end = ParallelDescriptor::second() - strt;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(end,IOProc);
if (ParallelDescriptor::IOProcessor()) {
std::cout << "Castro::reflux() at level " << level << " : time = " << end << std::endl;
std::cout << "Castro::reflux() at level " << llevel
<< " : time = " << end << std::endl;
}
#ifdef BL_LAZY
});
Expand Down
30 changes: 19 additions & 11 deletions Source/gravity/Castro_gravity.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,15 +122,17 @@ Castro::construct_old_gravity (Real time)

if (verbose > 0)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::construct_old_gravity() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::construct_old_gravity() time = " << run_time << " on level "
<< llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down Expand Up @@ -278,15 +280,17 @@ Castro::construct_new_gravity (Real time)

if (verbose > 0)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::construct_new_gravity() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::construct_new_gravity() time = " << run_time
<< " on level " << llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down Expand Up @@ -431,14 +435,16 @@ void Castro::construct_old_gravity_source(MultiFab& source, MultiFab& state_in,
if (castro::verbose > 1)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::construct_old_gravity_source() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::construct_old_gravity_source() time = " << run_time
<< " on level " << llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down Expand Up @@ -667,15 +673,17 @@ void Castro::construct_new_gravity_source(MultiFab& source, MultiFab& state_old,

if (castro::verbose > 1)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::construct_new_gravity_source() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::construct_new_gravity_source() time = " << run_time
<< " on level " << llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down
6 changes: 4 additions & 2 deletions Source/gravity/Gravity.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,13 +457,15 @@ Gravity::solve_for_phi (int level,
if (gravity::verbose)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real end = ParallelDescriptor::second() - strt;
amrex::Real end = ParallelDescriptor::second() - strt;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(end,IOProc);
amrex::Print() << "Gravity::solve_for_phi() time = " << end << " on level " << level << std::endl << std::endl;
amrex::Print() << "Gravity::solve_for_phi() time = " << end << " on level "
<< llevel << std::endl << std::endl;
#ifdef BL_LAZY
});
#endif
Expand Down
11 changes: 7 additions & 4 deletions Source/hydro/Castro_ctu_hydro.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1419,12 +1419,13 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co

#ifdef RADIATION
if (radiation->verbose>=1) {
amrex::Real llevel = level;
#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceIntMax(nstep_fsp, ParallelDescriptor::IOProcessorNumber());
if (ParallelDescriptor::IOProcessor() && nstep_fsp > 0) {
std::cout << "Radiation f-space advection on level " << level
std::cout << "Radiation f-space advection on level " << llevel
<< " takes as many as " << nstep_fsp;
if (nstep_fsp == 1) {
std::cout<< " substep.\n";
Expand Down Expand Up @@ -1482,15 +1483,17 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co

if (verbose > 0)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::construct_ctu_hydro_source() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::construct_ctu_hydro_source() time = " << run_time
<< " on level " << llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down
8 changes: 4 additions & 4 deletions Source/hydro/Castro_hybrid.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ Castro::construct_old_hybrid_source(MultiFab& source, MultiFab& state_old, Real

if (verbose > 1)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
Expand Down Expand Up @@ -65,8 +65,8 @@ Castro::construct_new_hybrid_source(MultiFab& source, MultiFab& state_old, Multi

if (verbose > 1)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
Expand Down
4 changes: 2 additions & 2 deletions Source/hydro/Castro_mol_hydro.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -743,8 +743,8 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update)

if (verbose > 0)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
Expand Down
8 changes: 4 additions & 4 deletions Source/problems/Castro_problem_source.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ Castro::construct_old_ext_source(MultiFab& source, MultiFab& state_in, Real time

if (verbose > 1)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
Expand Down Expand Up @@ -95,8 +95,8 @@ Castro::construct_new_ext_source(MultiFab& source, MultiFab& state_old, MultiFab

if (verbose > 1)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
Expand Down
14 changes: 9 additions & 5 deletions Source/reactions/Castro_react.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,14 +449,16 @@ Castro::react_state(MultiFab& s, MultiFab& r, Real time, Real dt, const int stra
if (verbose > 0)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::react_state() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::react_state() time = " << run_time
<< " on level " << llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down Expand Up @@ -852,15 +854,17 @@ Castro::react_state(Real time, Real dt)

amrex::Print() << "... Leaving burner on level " << level << " after completing full timestep of burning." << std::endl << std::endl;

const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time, IOProc);

amrex::Print() << "Castro::react_state() time = " << run_time << " on level " << level << std::endl << std::endl;
amrex::Print() << "Castro::react_state() time = " << run_time
<< " on level " << llevel << std::endl << std::endl;
#ifdef BL_LAZY
});
#endif
Expand Down
6 changes: 4 additions & 2 deletions Source/rotation/Castro_rotation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,16 @@ Castro::construct_old_rotation_source(MultiFab& source, MultiFab& state_in, Real
if (verbose > 1)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::construct_old_rotation_source() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::construct_old_rotation_source() time = " << run_time
<< " on level " << llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down
4 changes: 2 additions & 2 deletions Source/sources/Castro_geom.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ Castro::construct_old_geom_source(MultiFab& source, MultiFab& state_in, Real tim

if (verbose > 1)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
Expand Down
13 changes: 8 additions & 5 deletions Source/sources/Castro_sources.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,14 +159,16 @@ Castro::do_old_sources(
if (verbose > 0)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::do_old_sources() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::do_old_sources() time = " << run_time
<< " on level " << llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down Expand Up @@ -242,15 +244,16 @@ Castro::do_new_sources(

if (verbose > 0)
{
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Real run_time = ParallelDescriptor::second() - strt_time;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
amrex::Real run_time = ParallelDescriptor::second() - strt_time;
amrex::Real llevel = level;

#ifdef BL_LAZY
Lazy::QueueReduction( [=] () mutable {
#endif
ParallelDescriptor::ReduceRealMax(run_time,IOProc);

amrex::Print() << "Castro::do_new_sources() time = " << run_time << " on level " << level << "\n" << "\n";
amrex::Print() << "Castro::do_new_sources() time = " << run_time << " on level " << llevel << "\n" << "\n";
#ifdef BL_LAZY
});
#endif
Expand Down

0 comments on commit bb30ff4

Please sign in to comment.