Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BL/TinyProfiler SyncBeforeComms #2762

Closed
wants to merge 25 commits into from
Closed

BL/TinyProfiler SyncBeforeComms #2762

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
f236e57
First draft.
kngott May 10, 2022
c6fd925
Base Profile Fixes.
kngott May 11, 2022
8bafe67
Adjust timers and syncs for consistency.
kngott May 11, 2022
bea31cb
Extra lines.
kngott May 11, 2022
088771c
Take into account leaving finishes early.
kngott May 12, 2022
08638fc
Add HDF5 H5Z-ZFP support in CMake (#2753)
jrood-nrel May 11, 2022
cdc0daa
add scomp and ncomp arguments to IntegratorOps functions. (#2759)
dwillcox May 11, 2022
102e93c
Avoid the use of null stream (#2754)
WeiqunZhang May 11, 2022
a4bf36e
multilevel version of writeplotfiletoascii (#2742)
ajnonaka May 12, 2022
60a5547
Fix the Advection_AmrCore test (#2761)
WeiqunZhang May 12, 2022
45f9617
Time step in the AmrLevel test (#2763)
WeiqunZhang May 12, 2022
ec497e9
Fix Wstringop-overflow warning in FabConv (#2767)
WeiqunZhang May 12, 2022
6a8011c
this updates to recent Hypre API changes (#2765)
PaulMullowney May 12, 2022
b6a4e64
Fix maybe-uninitialized warning in calling mlock (#2768)
WeiqunZhang May 13, 2022
80a15e4
Update particle << operator after changes to id/cpu (#2769)
atmyers May 13, 2022
92ace57
Fix: AmrCore Move (#2773)
ax3l May 16, 2022
91124b1
Add an optional volume weighting to AMRErrorTag (#2772)
maximumcats May 16, 2022
2ab2135
Change repo html address to Ubuntu 20.04 (#2766)
etpalmer63 May 16, 2022
e04edae
CI--HIP: wget gpg key from https instead of http (#2771)
etpalmer63 May 16, 2022
5df7ff4
configure value of AMReX_GPU_RDC flag for use in cmake find_package(A…
etpalmer63 May 16, 2022
7bbe9fa
Fix the bug in the CMake build with AMReX_BASE_PROFILE. (#2774)
hengjiew May 16, 2022
32b1a0b
AmrCore: Include utility (#2778)
ax3l May 17, 2022
899d907
Add some timestep controls to the AMReX TimeIntegrator class for its …
dwillcox May 18, 2022
a0f5b04
Remove a Sync object.
kngott May 19, 2022
42a189e
Back to before
kngott May 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion Src/Base/AMReX_BLProfiler.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,24 @@ private:
std::string regname;
};

class BLProfileSync {

friend amrex::BLProfiler;

public:
static void Sync() noexcept;
static void Sync(const std::string& name) noexcept;
static void Sync(const char* name) noexcept;

static void StartSyncRegion() noexcept;
static void StartSyncRegion(const std::string& name) noexcept;
static void StartSyncRegion(const char* name) noexcept;
static void EndSyncRegion() noexcept;

private:
static int sync_counter;
static int use_prof_syncs;
};

namespace BLProfilerUtils {
void WriteHeader(std::ostream &os, const int colWidth,
Expand Down Expand Up @@ -426,7 +444,11 @@ inline std::string BLProfiler::CommStats::CFTToString(CommFuncType cft) {
bl_profiler_##rvname.start();
#define BL_PROFILE_REGION_VAR_STOP(fname, rvname) bl_profiler_##rvname.stop(); \
amrex::BLProfiler::RegionStop(fname);

#define BL_PROFILE_SYNC() amrex::BLProfileSync::Sync()
#define BL_PROFILE_SYNC_TIMED(fname) amrex::BLProfileSync::Sync()
#define BL_PROFILE_SYNC_START() amrex::BLProfileSync::StartSyncRegion()
#define BL_PROFILE_SYNC_START_TIMED(fname) amrex::BLProfileSync::StartSyncRegion(fname)
#define BL_PROFILE_SYNC_STOP() amrex::BLProfileSync::EndSyncRegion()
#define BL_PROFILE_TINY_FLUSH()
#define BL_PROFILE_FLUSH() { amrex::BLProfiler::Finalize(true); }

Expand Down Expand Up @@ -508,6 +530,13 @@ inline std::string BLProfiler::CommStats::CFTToString(CommFuncType cft) {
#define BL_PROFILE_REGION_VAR(fname, rvname)
#define BL_PROFILE_REGION_VAR_START(fname, rvname)
#define BL_PROFILE_REGION_VAR_STOP(fname, rvname)

#define BL_PROFILE_SYNC() amrex::TinyProfileSync::Sync()
#define BL_PROFILE_SYNC_TIMED(fname) amrex::TinyProfileSync::Sync()
#define BL_PROFILE_SYNC_START() amrex::TinyProfileSync::StartSyncRegion()
#define BL_PROFILE_SYNC_START_TIMED(fname) amrex::TinyProfileSync::StartSyncRegion(fname)
#define BL_PROFILE_SYNC_STOP() amrex::TinyProfileSync::EndSyncRegion()

#define BL_PROFILE_TINY_FLUSH() amrex::TinyProfiler::Finalize(true)
#define BL_PROFILE_FLUSH()
#define BL_TRACE_PROFILE_FLUSH()
Expand Down Expand Up @@ -568,6 +597,11 @@ class BLProfiler
#define BL_PROFILE_REGION_VAR(fname, rvname)
#define BL_PROFILE_REGION_VAR_START(fname, rvname)
#define BL_PROFILE_REGION_VAR_STOP(fname, rvname)
#define BL_PROFILE_SYNC()
#define BL_PROFILE_SYNC_TIMED(fname)
#define BL_PROFILE_SYNC_START()
#define BL_PROFILE_SYNC_START_TIMED(fname)
#define BL_PROFILE_SYNC_STOP()
#define BL_PROFILE_TINY_FLUSH()
#define BL_PROFILE_FLUSH()
#define BL_TRACE_PROFILE_FLUSH()
Expand Down
88 changes: 81 additions & 7 deletions Src/Base/AMReX_BLProfiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ int BLProfiler::CallStats::cstatsVersion(1);
Vector<BLProfiler::CallStatsStack> BLProfiler::callIndexStack;
Vector<BLProfiler::CallStatsPatch> BLProfiler::callIndexPatch;

int BLProfileSync::use_prof_syncs = 0;
int BLProfileSync::sync_counter = 0;

#ifdef BL_TRACE_PROFILING
int BLProfiler::callStackDepth(-1);
int BLProfiler::prevCallStackDepth(0);
Expand Down Expand Up @@ -271,13 +274,20 @@ void BLProfiler::Initialize() {


void BLProfiler::InitParams() {
ParmParse pParse("blprofiler");
pParse.queryAdd("prof_nfiles", nProfFiles);
pParse.queryAdd("prof_csflushsize", csFlushSize);
pParse.queryAdd("prof_traceflushsize", traceFlushSize);
pParse.queryAdd("prof_flushinterval", flushInterval);
pParse.queryAdd("prof_flushtimeinterval", flushTimeInterval);
pParse.queryAdd("prof_flushprint", bFlushPrint);
{
ParmParse pParse("blprofiler");
pParse.queryAdd("prof_nfiles", nProfFiles);
pParse.queryAdd("prof_csflushsize", csFlushSize);
pParse.queryAdd("prof_traceflushsize", traceFlushSize);
pParse.queryAdd("prof_flushinterval", flushInterval);
pParse.queryAdd("prof_flushtimeinterval", flushTimeInterval);
pParse.queryAdd("prof_flushprint", bFlushPrint);
}

{
ParmParse pParse("amrex");
pParse.queryAdd("use_profiler_syncs", BLProfileSync::use_prof_syncs);
}
}


Expand Down Expand Up @@ -1546,6 +1556,70 @@ void BLProfiler::CommStats::UnFilter(CommFuncType cft) {
}
}

void
BLProfileSync::Sync () noexcept
{
if (use_prof_syncs)
{ ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub()); }
}

void
BLProfileSync::Sync (const std::string& name) noexcept
{
if (use_prof_syncs) {
BLProfiler synctimer(name);
ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub());
}
}

void
BLProfileSync::Sync (const char* name) noexcept
{
if (use_prof_syncs) {
BLProfiler synctimer(name);
ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub());
}
}

void
BLProfileSync::StartSyncRegion () noexcept
{
if (use_prof_syncs) {
if (sync_counter == 0) {
ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub());
}
sync_counter++;
}
}

void
BLProfileSync::StartSyncRegion (const std::string& name) noexcept {
if (use_prof_syncs) {
if (sync_counter == 0) {
BLProfiler synctimer(name);
ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub());
}
sync_counter++;
}
}

void
BLProfileSync::StartSyncRegion (const char* name) noexcept {
if (use_prof_syncs) {
if (sync_counter == 0) {
BLProfiler synctimer(name);
ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub());
}
sync_counter++;
}
}

void
BLProfileSync::EndSyncRegion () noexcept {
if (use_prof_syncs) {
sync_counter--;
}
}

namespace {
const int EOS(-1);
Expand Down
6 changes: 2 additions & 4 deletions Src/Base/AMReX_FabArray.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2696,7 +2696,7 @@ template <class FAB>
void
FabArray<FAB>::FillBoundaryAndSync (const Periodicity& period)
{
BL_PROFILE("FAbArray::FillBoundaryAndSync()");
BL_PROFILE("FabArray::FillBoundaryAndSync()");
if (n_grow.max() > 0 || !is_cell_centered()) {
FillBoundaryAndSync_nowait(0, nComp(), n_grow, period);
FillBoundaryAndSync_finish();
Expand All @@ -2708,7 +2708,7 @@ void
FabArray<FAB>::FillBoundaryAndSync (int scomp, int ncomp, const IntVect& nghost,
const Periodicity& period)
{
BL_PROFILE("FAbArray::FillBoundaryAndSync()");
BL_PROFILE("FabArray::FillBoundaryAndSync()");
if (nghost.max() > 0 || !is_cell_centered()) {
FillBoundaryAndSync_nowait(scomp, ncomp, nghost, period);
FillBoundaryAndSync_finish();
Expand Down Expand Up @@ -2908,7 +2908,6 @@ template <typename BUF>
void
FabArray<FAB>::FillBoundary_nowait (int scomp, int ncomp, const Periodicity& period, bool cross)
{
BL_PROFILE("FillBoundary_nowait()");
FBEP_nowait<BUF>(scomp, ncomp, nGrowVect(), period, cross);
}

Expand All @@ -2918,7 +2917,6 @@ void
FabArray<FAB>::FillBoundary_nowait (int scomp, int ncomp, const IntVect& nghost,
const Periodicity& period, bool cross)
{
BL_PROFILE("FillBoundary_nowait()");
FBEP_nowait<BUF>(scomp, ncomp, nghost, period, cross);
}

Expand Down
9 changes: 8 additions & 1 deletion Src/Base/AMReX_FabArrayCommI.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ FabArray<FAB>::FBEP_nowait (int scomp, int ncomp, const IntVect& nghost,
bool enforce_periodicity_only,
bool override_sync)
{
BL_PROFILE_SYNC_START_TIMED("SyncBeforeComms");
BL_PROFILE("FillBoundary_nowait()");

AMREX_ASSERT_WITH_MESSAGE(!fbd, "FillBoundary_nowait() called when comm operation already in progress.");
AMREX_ASSERT(!enforce_periodicity_only || !override_sync);

Expand Down Expand Up @@ -240,6 +243,8 @@ FabArray<FAB>::FillBoundary_finish ()
fbd.reset();

#endif

BL_PROFILE_SYNC_STOP();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might too late because the function might return early.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be fixed.

}

template <class FAB>
Expand All @@ -255,7 +260,6 @@ FabArray<FAB>::ParallelCopy (const FabArray<FAB>& src,
const FabArrayBase::CPC * a_cpc)
{
BL_PROFILE("FabArray::ParallelCopy()");

ParallelCopy_nowait(src, scomp, dcomp, ncomp, snghost, dnghost, period, op, a_cpc);
ParallelCopy_finish();
}
Expand Down Expand Up @@ -312,6 +316,7 @@ FabArray<FAB>::ParallelCopy_nowait (const FabArray<FAB>& src,
const FabArrayBase::CPC * a_cpc,
bool to_ghost_cells_only)
{
BL_PROFILE_SYNC_START_TIMED("SyncBeforeComms");
BL_PROFILE("FabArray::ParallelCopy_nowait()");

AMREX_ASSERT_WITH_MESSAGE(!pcd, "ParallelCopy_nowait() called when comm operation already in progress.");
Expand Down Expand Up @@ -559,6 +564,8 @@ FabArray<FAB>::ParallelCopy_finish ()

#endif /*BL_USE_MPI*/

BL_PROFILE_SYNC_STOP();

}

template <class FAB>
Expand Down
20 changes: 20 additions & 0 deletions Src/Base/AMReX_TinyProfiler.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,5 +111,25 @@ private:
TinyProfiler tprof;
};


class TinyProfileSync {

friend amrex::TinyProfiler;

public:
static void Sync() noexcept;
static void Sync(const std::string& name) noexcept;
static void Sync(const char* name) noexcept;

static void StartSyncRegion() noexcept;
static void StartSyncRegion(const std::string& name) noexcept;
static void StartSyncRegion(const char* name) noexcept;
static void EndSyncRegion() noexcept;

private:
static int sync_counter;
static int use_prof_syncs;
};

}
#endif
71 changes: 71 additions & 0 deletions Src/Base/AMReX_TinyProfiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ double TinyProfiler::t_init = std::numeric_limits<double>::max();
int TinyProfiler::device_synchronize_around_region = 0;
int TinyProfiler::n_print_tabs = 0;
int TinyProfiler::verbose = 0;
int TinyProfileSync::use_prof_syncs = 0;
int TinyProfileSync::sync_counter = 0;

namespace {
std::set<std::string> improperly_nested_timers;
Expand Down Expand Up @@ -304,6 +306,10 @@ TinyProfiler::Initialize () noexcept
pp.queryAdd("verbose", verbose);
pp.queryAdd("v", verbose);
}
{
amrex::ParmParse pp("amrex");
pp.queryAdd("use_profiler_syncs", TinyProfileSync::use_prof_syncs);
}
}

void
Expand Down Expand Up @@ -638,4 +644,69 @@ TinyProfiler::PrintCallStack (std::ostream& os)
}
}

void
TinyProfileSync::Sync () noexcept
{
if (use_prof_syncs)
{ ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub()); }
}

void
TinyProfileSync::Sync (const std::string& name) noexcept
{
if (use_prof_syncs) {
TinyProfiler synctimer(name);
ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub());
}
}

void
TinyProfileSync::Sync (const char* name) noexcept
{
if (use_prof_syncs) {
TinyProfiler synctimer(name);
ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub());
}
}

void
TinyProfileSync::StartSyncRegion () noexcept
{
if (use_prof_syncs) {
if (sync_counter == 0) {
ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub());
}
sync_counter++;
}
}

void
TinyProfileSync::StartSyncRegion (const std::string& name) noexcept {
if (use_prof_syncs) {
if (sync_counter == 0) {
TinyProfiler synctimer(name);
ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub());
}
sync_counter++;
}
}

void
TinyProfileSync::StartSyncRegion (const char* name) noexcept {
if (use_prof_syncs) {
if (sync_counter == 0) {
TinyProfiler synctimer(name);
ParallelDescriptor::Barrier(ParallelContext::CommunicatorSub());
}
sync_counter++;
}
}

void
TinyProfileSync::EndSyncRegion () noexcept {
if (use_prof_syncs) {
sync_counter--;
}
}

}