From e8ee2a75b576d2953e048ae8f7a3996f8164c3f2 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 4 May 2021 20:23:39 -0400 Subject: [PATCH 1/2] set ssc threading default to true --- source/adios2/engine/ssc/SscReader.h | 2 +- source/adios2/engine/ssc/SscWriter.cpp | 30 +++++++++++--------------- source/adios2/engine/ssc/SscWriter.h | 1 - 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index 71c198dee4..7a19bcb5cc 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -100,7 +100,7 @@ class SscReader : public Engine int m_Verbosity = 0; int m_OpenTimeoutSecs = 10; - bool m_Threading = false; + bool m_Threading = true; }; } // end namespace engine diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp index f9089103e6..95b76943d6 100644 --- a/source/adios2/engine/ssc/SscWriter.cpp +++ b/source/adios2/engine/ssc/SscWriter.cpp @@ -33,17 +33,14 @@ SscWriter::SscWriter(IO &io, const std::string &name, const Mode mode, int providedMpiMode; MPI_Query_thread(&providedMpiMode); - if (providedMpiMode != MPI_THREAD_MULTIPLE) + if (m_Threading && providedMpiMode != MPI_THREAD_MULTIPLE) { - if (m_Threading == true) + m_Threading = false; + if (m_WriterRank == 0 && m_Verbosity > 0) { - m_Threading = false; - if (m_WriterRank == 0) - { - std::cout << "SSC Threading disabled as MPI is not initialized " - "with multi-threads" - << std::endl; - } + std::cout << "SSC Threading disabled as MPI is not initialized " + "with multi-threads" + << std::endl; } } @@ -83,7 +80,9 @@ StepStatus SscWriter::BeginStep(StepMode mode, const float timeoutSeconds) { if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked) { - MpiWait(); + MPI_Waitall(static_cast(m_MpiRequests.size()), + m_MpiRequests.data(), MPI_STATUSES_IGNORE); + m_MpiRequests.clear(); } else { @@ -173,13 +172,6 @@ void SscWriter::EndStep() void SscWriter::Flush(const int transportIndex) { TAU_SCOPED_TIMER_FUNC(); } -void SscWriter::MpiWait() -{ - MPI_Waitall(static_cast(m_MpiRequests.size()), m_MpiRequests.data(), - MPI_STATUSES_IGNORE); - m_MpiRequests.clear(); -} - void SscWriter::SyncMpiPattern() { TAU_SCOPED_TIMER_FUNC(); @@ -356,7 +348,9 @@ void SscWriter::DoClose(const int transportIndex) { if (m_CurrentStep > 0) { - MpiWait(); + MPI_Waitall(static_cast(m_MpiRequests.size()), + m_MpiRequests.data(), MPI_STATUSES_IGNORE); + m_MpiRequests.clear(); } m_Buffer[0] = 1; diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h index 82a0148912..343a84d3c8 100644 --- a/source/adios2/engine/ssc/SscWriter.h +++ b/source/adios2/engine/ssc/SscWriter.h @@ -66,7 +66,6 @@ class SscWriter : public Engine void SyncMpiPattern(); void SyncWritePattern(bool finalStep = false); void SyncReadPattern(); - void MpiWait(); void EndStepFirst(); void EndStepConsequentFixed(); void EndStepConsequentFlexible(); From d14612c813323542d86fb6c3aa9034dec9cacd46 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 7 May 2021 10:56:49 -0400 Subject: [PATCH 2/2] updated ssc doc --- docs/user_guide/source/engines/ssc.rst | 2 +- source/adios2/engine/ssc/SscReader.cpp | 22 ++++++++-------------- source/adios2/engine/ssc/SscReader.h | 4 +--- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/docs/user_guide/source/engines/ssc.rst b/docs/user_guide/source/engines/ssc.rst index 68ae308be0..7738459d09 100644 --- a/docs/user_guide/source/engines/ssc.rst +++ b/docs/user_guide/source/engines/ssc.rst @@ -8,7 +8,7 @@ The SSC engine takes the following parameters: 1. ``OpenTimeoutSecs``: Default **10**. Timeout in seconds for opening a stream. The SSC engine's open function will block until the RendezvousAppCount is reached, or timeout, whichever comes first. If it reaches the timeout, SSC will throw an exception. -2. ``Threading``: Default **False**. SSC will use threads to hide the time cost for metadata manipulation and data transfer when this parameter is set to **true**. SSC will check if MPI is initialized with multi-thread enabled, and if not, then SSC will force this parameter to be **false**. Please do NOT enable threading when multiple I/O streams are opened in an application, as it will cause unpredictable errors. +2. ``Threading``: Default **False**. SSC will use threads to hide the time cost for metadata manipulation and data transfer when this parameter is set to **true**. SSC will check if MPI is initialized with multi-thread enabled, and if not, then SSC will force this parameter to be **false**. Please do NOT enable threading when multiple I/O streams are opened in an application, as it will cause unpredictable errors. This parameter is only effective when writer definitions and reader selections are NOT locked. For cases definitions and reader selections are locked, SSC has a more optimized way to do data transfers, and thus it will not use this parameter. =============================== ================== ================================================ **Key** **Value Format** **Default** and Examples diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 3ff29fa189..33eaef4b2b 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -272,19 +272,12 @@ void SscReader::EndStepFirstFlexible() { MPI_Win_free(&m_MpiWin); SyncReadPattern(); -} - -void SscReader::EndStepConsequentFlexible() { MPI_Win_free(&m_MpiWin); } - -void SscReader::EndBeginStepFirstFlexible() -{ - EndStepFirstFlexible(); BeginStepFlexible(m_StepStatus); } -void SscReader::EndBeginStepConsequentFlexible() +void SscReader::EndStepConsequentFlexible() { - EndStepConsequentFlexible(); + MPI_Win_free(&m_MpiWin); BeginStepFlexible(m_StepStatus); } @@ -312,23 +305,24 @@ void SscReader::EndStep() if (m_Threading) { m_EndStepThread = - std::thread(&SscReader::EndBeginStepFirstFlexible, this); + std::thread(&SscReader::EndStepFirstFlexible, this); } else { - EndStepFirstFlexible(); + MPI_Win_free(&m_MpiWin); + SyncReadPattern(); } } else { if (m_Threading) { - m_EndStepThread = std::thread( - &SscReader::EndBeginStepConsequentFlexible, this); + m_EndStepThread = + std::thread(&SscReader::EndStepConsequentFlexible, this); } else { - EndStepConsequentFlexible(); + MPI_Win_free(&m_MpiWin); } } } diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index 7a19bcb5cc..1059f135f6 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -72,8 +72,6 @@ class SscReader : public Engine void EndStepFixed(); void EndStepFirstFlexible(); void EndStepConsequentFlexible(); - void EndBeginStepFirstFlexible(); - void EndBeginStepConsequentFlexible(); #define declare_type(T) \ void DoGetSync(Variable &, T *) final; \ @@ -100,7 +98,7 @@ class SscReader : public Engine int m_Verbosity = 0; int m_OpenTimeoutSecs = 10; - bool m_Threading = true; + bool m_Threading = false; }; } // end namespace engine