Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

normalized dataman monitor performance data for better learning #2733

Merged
merged 2 commits into from
May 31, 2021
Merged

normalized dataman monitor performance data for better learning #2733

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f5bbde2
normalized dataman monitor performance data for better learning
JasonRuonanWang May 29, 2021
6dd065d
further normalize dataman monitor data
JasonRuonanWang May 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 49 additions & 33 deletions source/adios2/engine/dataman/DataManMonitor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,11 +155,7 @@ void DataManMonitor::AddBytes(const size_t bytes)

void DataManMonitor::SetRequiredAccuracy(const std::string &accuracyRequired)
{
if (accuracyRequired.empty())
{
m_RequiredAccuracy = 0.00000000001;
}
else
if (!accuracyRequired.empty())
{
m_RequiredAccuracy = std::stof(accuracyRequired);
}
Expand All @@ -170,11 +166,7 @@ void DataManMonitor::AddCompression(const std::string &method,
{
m_CompressionMethod = method;

if (accuracyUsed.empty())
{
m_CompressionAccuracy = 0.00000000001;
}
else
if (!accuracyUsed.empty())
{
m_CompressionAccuracy = std::stof(accuracyUsed);
}
Expand Down Expand Up @@ -213,31 +205,23 @@ void DataManMonitor::OutputJson(const std::string &filename)
output["Info"]["RoundLatency"] = m_RoundLatency;
output["Info"]["ClockError"] = m_ClockError;

std::ofstream file;
file.open((filename + ".json").c_str(),
std::fstream::out | std::fstream::app);
bool fileExists = FileExisted(filename + ".json");

std::ofstream file((filename + ".json").c_str(),
std::fstream::out | std::fstream::app);
if (!fileExists)
{
file << "ADOPS2 DataMan performance measurements" << std::endl;
}
file << output.dump() << "\0" << std::endl;
file.close();
}

void DataManMonitor::OutputCsv(const std::string &filename)
{

bool fileExists;
std::ifstream checkFile((filename + ".csv").c_str());
if (checkFile.is_open())
{
fileExists = true;
}
else
{
fileExists = false;
}
checkFile.close();

std::ofstream file;
file.open((filename + ".csv").c_str(),
std::fstream::out | std::fstream::app);
bool fileExists = FileExisted(filename + ".csv");
std::ofstream file((filename + ".csv").c_str(),
std::fstream::out | std::fstream::app);
if (!fileExists)
{
file << "bandwidth, latency, precision, completeness, size, "
Expand All @@ -248,17 +232,49 @@ void DataManMonitor::OutputCsv(const std::string &filename)
file << floor(log2(m_AccumulatedLatency /
static_cast<double>(m_CurrentStep + 1)))
<< ", ";
file << floor(log10(m_RequiredAccuracy)) << ", ";
if (m_RequiredAccuracy == 0)
{
file << 0 << ", ";
}
else if (m_RequiredAccuracy < 0.00001)
{
file << 1 << ", ";
}
else
{
file << round(log10(m_RequiredAccuracy)) + 6 << ", ";
}
file << ceil(log2(m_DropRate * 100 + 1)) << ", ";
file << floor(log2(m_StepBytes)) << ", ";
file << ceil(log2(m_CombiningSteps + 1)) << ", ";
file << floor(log10(m_CompressionAccuracy)) << ", ";
file << floor(log(m_StepBytes) / log(5)) - 4 << ", ";
file << ceil(log2(m_CombiningSteps)) << ", ";
if (m_CompressionAccuracy == 0)
{
file << 0 << ", ";
}
else if (m_CompressionAccuracy < 0.00001)
{
file << 1 << ", ";
}
else
{
file << round(log10(m_CompressionAccuracy)) + 6 << ", ";
}
file << static_cast<int>(m_WriterThreading) * 2 +
static_cast<int>(m_ReaderThreading)
<< std::endl;
file.close();
}

bool DataManMonitor::FileExisted(const std::string &filename)
{
std::ifstream checkFile(filename.c_str());
if (checkFile.is_open())
{
return true;
}
return false;
}

} // end namespace engine
} // end namespace core
} // end namespace adios2
6 changes: 4 additions & 2 deletions source/adios2/engine/dataman/DataManMonitor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ class DataManMonitor
void OutputCsv(const std::string &filename);

private:
bool FileExisted(const std::string &filename);

using TimePoint = std::chrono::time_point<std::chrono::system_clock>;
TimePoint m_InitialTimer;
std::queue<TimePoint> m_StepTimers;
Expand All @@ -67,8 +69,8 @@ class DataManMonitor
double m_StepsPerSecond = 0;
double m_AccumulatedLatency = 0;
std::string m_CompressionMethod;
float m_CompressionAccuracy;
float m_RequiredAccuracy;
float m_CompressionAccuracy = 0;
float m_RequiredAccuracy = 0.00001;
std::string m_TransportMethod;
bool m_ReaderThreading = false;
bool m_WriterThreading = false;
Expand Down