From 734e7f87289bdded8ca65adcb0e8b96ceb924f01 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 28 Mar 2024 16:38:27 -0400 Subject: [PATCH 01/11] Add support for user options in ~/.config/adios2/adios2.yaml Currently supported options: General: verbose: 0 Campaign: active: true hostname: LAP131864 campaignstorepath: ~/dropbox/adios-campaign-store cachepath: /tmp/campaign verbose: 0 SST: verbose: 0 ~ is processed to $HOME (%HOMEPATH% on Windows) (but only in select path options) Usage: every engine has "const UserOptions &m_UserOption" member available and can go from there. Other parts of the code can call "static const UserOptions ADIOS::GetUserOptions()" and go from there. Make Campaign AUTO (on when sqlite3 and zlib are found) - still the campaign manager can be deactivated in adios2.yaml --- CMakeLists.txt | 2 +- source/adios2/common/ADIOSTypes.h | 27 ++++ source/adios2/core/ADIOS.cpp | 39 +++++- source/adios2/core/ADIOS.h | 5 + source/adios2/core/Engine.cpp | 3 +- source/adios2/core/Engine.h | 3 + .../engine/campaign/CampaignManager.cpp | 32 ++--- .../adios2/engine/campaign/CampaignManager.h | 7 +- .../adios2/engine/campaign/CampaignReader.cpp | 110 ++++++++-------- .../adios2/engine/campaign/CampaignReader.h | 6 +- source/adios2/engine/sst/SstParamParser.cpp | 5 +- source/adios2/engine/sst/SstParamParser.h | 4 +- source/adios2/engine/sst/SstReader.cpp | 2 +- source/adios2/engine/sst/SstWriter.cpp | 2 +- source/adios2/helper/adiosUserOptions.cpp | 121 ++++++++++++++++++ source/adios2/helper/adiosUserOptions.h | 30 +++++ source/adios2/helper/adiosYAML.cpp | 108 ++++++++++++++++ source/adios2/helper/adiosYAML.h | 4 + source/utils/bpls/bpls.cpp | 51 ++++++-- source/utils/bpls/bpls.h | 3 +- 20 files changed, 455 insertions(+), 109 deletions(-) create mode 100644 source/adios2/helper/adiosUserOptions.cpp create mode 100644 source/adios2/helper/adiosUserOptions.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8511e152bf..37e4b6016b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,7 +183,7 @@ adios_option(Profiling "Enable support for profiling" AUTO) adios_option(Endian_Reverse "Enable support for Little/Big Endian Interoperability" AUTO) adios_option(Sodium "Enable support for Sodium for encryption" AUTO) adios_option(Catalyst "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO) -adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB)" OFF) +adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB)" AUTO) adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF) adios_option(Derived_Variable "Enable support for derived variables" OFF) adios_option(PIP "Enable support for pip packaging" OFF) diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 5d9eeb2624..fad51e8fd9 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -357,6 +357,33 @@ std::string ToString(const Dims &dims); std::string ToString(const Box &box); std::string ToString(const MemorySpace value); +/** UserOptions holds all user options from ~/.config/adios2/adios2.yaml */ +struct UserOptions +{ + struct General + { + int verbose; + }; + + struct Campaign + { + bool active; + int verbose; + std::string hostname; + std::string campaignstorepath; + std::string cachepath; + }; + + struct SST + { + int verbose; + }; + + General general; + Campaign campaign; + SST sst; +}; + /** * os << [adios2_type] enables output of adios2 enums/classes directly * to output streams (e.g. std::cout), if ToString() can handle [adios2_type]. diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 6504474d81..a77ad1d0fc 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -19,6 +19,7 @@ #include "adios2/core/IO.h" #include "adios2/helper/adiosCommDummy.h" #include "adios2/helper/adiosFunctions.h" //InquireKey, BroadcastFile +#include "adios2/helper/adiosYAML.h" #include "adios2/operator/OperatorFactory.h" #include @@ -106,6 +107,10 @@ std::mutex PerfStubsMutex; static std::atomic_uint adios_refcount(0); // adios objects at the same time static std::atomic_uint adios_count(0); // total adios objects during runtime +/** User defined options from ~/.config/adios2/adios2.yaml if it exists */ +static adios2::UserOptions UserOptions; +const adios2::UserOptions &ADIOS::GetUserOptions() { return UserOptions; }; + ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string hostLanguage) : m_HostLanguage(hostLanguage), m_Comm(std::move(comm)), m_ConfigFile(configFile), m_CampaignManager(m_Comm) @@ -124,6 +129,7 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string } } #endif + ProcessUserConfig(); if (!configFile.empty()) { if (!adios2sys::SystemTools::FileExists(configFile)) @@ -143,8 +149,11 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string #ifdef ADIOS2_HAVE_KOKKOS m_GlobalServices.Init_Kokkos_API(); #endif - std::string campaignName = "campaign_" + std::to_string(adios_count); - m_CampaignManager.Open(campaignName); + if (UserOptions.campaign.active) + { + std::string campaignName = "campaign_" + std::to_string(adios_count); + m_CampaignManager.Open(campaignName, UserOptions); + } } ADIOS::ADIOS(const std::string configFile, const std::string hostLanguage) @@ -166,7 +175,26 @@ ADIOS::~ADIOS() { m_GlobalServices.Finalize(); } - m_CampaignManager.Close(); + if (UserOptions.campaign.active) + { + m_CampaignManager.Close(); + } +} + +void ADIOS::ProcessUserConfig() +{ + // read config parameters from config file + std::string homePath; +#ifdef _WIN32 + homePath = getenv("HOMEPATH"); +#else + homePath = getenv("HOME"); +#endif + const std::string cfgFile = homePath + "/.config/adios2/adios2.yaml"; + if (adios2sys::SystemTools::FileExists(cfgFile)) + { + helper::ParseUserOptionsFile(m_Comm, cfgFile, UserOptions, homePath); + } } IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder) @@ -330,7 +358,10 @@ void ADIOS::YAMLInitIO(const std::string &configFileYAML, const std::string &con void ADIOS::RecordOutputStep(const std::string &name, const size_t step, const double time) { - m_CampaignManager.Record(name, step, time); + if (UserOptions.campaign.active) + { + m_CampaignManager.Record(name, step, time); + } } void ADIOS::Global_init_AWS_API() diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index bf534a960e..4dd14e17d7 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -164,6 +164,9 @@ class ADIOS void RecordOutputStep(const std::string &name, const size_t step = UnknownStep, const double time = UnknownTime); + /** A constant reference to the user options from ~/.config/adios2/adios2.yaml */ + static const adios2::UserOptions &GetUserOptions(); + private: /** Communicator given to parallel constructor. */ helper::Comm m_Comm; @@ -204,6 +207,8 @@ class ADIOS void YAMLInitIO(const std::string &configFileYAML, const std::string &configFileContents, core::IO &io); + void ProcessUserConfig(); + private: /* Global services that we want to initialize at most once and shutdown automatically when the ADIOS object is destructed. This only works diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index ab0a5e1761..72ab54d4ac 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -23,7 +23,8 @@ namespace core Engine::Engine(const std::string engineType, IO &io, const std::string &name, const Mode openMode, helper::Comm comm) -: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm)) +: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm)), + m_UserOptions(io.m_ADIOS.GetUserOptions()) { m_FailVerbose = (m_Comm.Rank() == 0); } diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index 5e2a721cae..0ab6ed4a00 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -505,6 +505,9 @@ class Engine * if no communicator is passed */ helper::Comm m_Comm; + /** User options parsed by the ADIOS object, here just for easy reference */ + const UserOptions &m_UserOptions; + /** keeps track of current advance status */ StepStatus m_AdvanceStatus = StepStatus::OK; diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index bd511ce687..3375464d6b 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -43,7 +43,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name "SQL error on writing records:"); sqlite3_free(zErrMsg); } - sqlcmd = "CREATE TABLE if not exists bpfiles (name);"; + sqlcmd = "CREATE TABLE if not exists bpfiles (name PRIMARY KEY);"; rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); if (rc != SQLITE_OK) { @@ -56,7 +56,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name for (auto &r : cmap) { - sqlcmd = "INSERT INTO bpfiles (name)\n"; + sqlcmd = "INSERT OR IGNORE INTO bpfiles (name)\n"; sqlcmd += "VALUES('" + r.first + "');"; rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg); if (rc != SQLITE_OK) @@ -74,31 +74,27 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name return 0; } -CampaignManager::CampaignManager(adios2::helper::Comm &comm) -{ - m_WriterRank = comm.Rank(); - if (m_Verbosity == 5) - { - std::cout << "Campaign Manager " << m_WriterRank << " constructor called" << std::endl; - } -} +CampaignManager::CampaignManager(adios2::helper::Comm &comm) { m_WriterRank = comm.Rank(); } CampaignManager::~CampaignManager() { - if (m_Verbosity == 5) - { - std::cout << "Campaign Manager " << m_WriterRank << " desctructor called\n"; - } if (m_Opened) { Close(); } } -void CampaignManager::Open(const std::string &name) +void CampaignManager::Open(const std::string &name, const UserOptions &options) { - m_Name = m_CampaignDir + "/" + name + "_" + std::to_string(m_WriterRank); - if (m_Verbosity == 5) + const UserOptions::Campaign &opts = options.campaign; + m_Options.active = opts.active; + m_Options.hostname = opts.hostname; + m_Options.campaignstorepath = opts.campaignstorepath; + m_Options.cachepath = opts.cachepath; + m_Options.verbose = opts.verbose; + + m_Name = m_CampaignDir + PathSeparator + name + "_" + std::to_string(m_WriterRank); + if (m_Options.verbose > 0) { std::cout << "Campaign Manager " << m_WriterRank << " Open(" << m_Name << ")\n"; } @@ -107,7 +103,7 @@ void CampaignManager::Open(const std::string &name) void CampaignManager::Record(const std::string &name, const size_t step, const double time) { - if (m_Verbosity == 5) + if (m_Options.verbose > 0) { std::cout << "Campaign Manager " << m_WriterRank << " Record name = " << name << " step = " << step << " time = " << time << "\n"; diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h index 7a313f5627..498de83caa 100644 --- a/source/adios2/engine/campaign/CampaignManager.h +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -16,6 +16,7 @@ #define ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ #include "adios2/common/ADIOSConfig.h" +#include "adios2/common/ADIOSTypes.h" #include "adios2/helper/adiosComm.h" #ifdef ADIOS2_HAVE_CAMPAIGN @@ -39,15 +40,15 @@ class CampaignManager CampaignManager(helper::Comm &comm); ~CampaignManager(); - void Open(const std::string &name); + void Open(const std::string &name, const UserOptions &options); void Record(const std::string &name, const size_t step, const double time); void Close(); private: + UserOptions::Campaign m_Options; bool m_Opened = false; std::string m_Name; int m_WriterRank; - int m_Verbosity = 0; CampaignRecordMap cmap; std::ofstream m_Output; const std::string m_CampaignDir = "adios-campaign"; @@ -56,7 +57,7 @@ class CampaignManager public: CampaignManager(helper::Comm &comm){}; ~CampaignManager() = default; - void Open(const std::string &name){}; + void Open(const std::string &name, const INIOptions &options){}; void Record(const std::string &name, const size_t step, const double time){}; void Close(){}; diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 29642d64dc..2bec32bf22 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -15,6 +15,7 @@ #include "adios2/helper/adiosNetwork.h" // GetFQDN #include "adios2/helper/adiosSystem.h" // CreateDirectory #include +#include #include #include @@ -33,7 +34,7 @@ CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, { m_ReaderRank = m_Comm.Rank(); Init(); - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " Open(" << m_Name << ") in constructor." << std::endl; @@ -44,7 +45,7 @@ CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, CampaignReader::~CampaignReader() { /* CampaignReader destructor does close and finalize */ - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " destructor on " << m_Name << "\n"; } @@ -61,7 +62,7 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, const float timeoutSec // so this forced increase should not be here ++m_CurrentStep; - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " BeginStep() new step " << m_CurrentStep << "\n"; @@ -88,7 +89,7 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, const float timeoutSec void CampaignReader::PerformGets() { - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " PerformGets()\n"; } @@ -106,7 +107,7 @@ void CampaignReader::EndStep() PerformGets(); } - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " EndStep()\n"; } @@ -116,49 +117,18 @@ void CampaignReader::EndStep() void CampaignReader::Init() { - // read config parameters from config file - const std::string cfgFile = "/.config/adios2/campaign.cfg"; - std::string homePath; -#ifdef _WIN32 - homePath = getenv("HOMEPATH"); -#else - homePath = getenv("HOME"); -#endif - ReadConfig(std::string(homePath + cfgFile)); InitParameters(); InitTransports(); } -void CampaignReader::ReadConfig(std::string configPath) -{ - std::ifstream fileStream(configPath); - - if (!fileStream) - { - return; - } - - std::ostringstream fileSS; - fileSS << fileStream.rdbuf(); - fileStream.close(); - size_t posEndline = 0; - size_t posSpace = 0; - std::string token; - std::string endline = "\n"; - std::string fileString = fileSS.str(); - while ((posEndline = fileString.find(endline)) != std::string::npos) - { - std::string line = fileString.substr(0, posEndline); - posSpace = fileString.find(" "); - std::string token1 = line.substr(0, posSpace); - std::string token2 = line.substr(posSpace + 1, line.size()); - // trim? - fileString.erase(0, posEndline + endline.length()); - } - return; -} void CampaignReader::InitParameters() { + const UserOptions::Campaign &opts = m_UserOptions.campaign; + m_Options.active = true; // this is really just for Recording + m_Options.hostname = opts.hostname; + m_Options.campaignstorepath = opts.campaignstorepath; + m_Options.cachepath = opts.cachepath; + m_Options.verbose = opts.verbose; for (const auto &pair : m_IO.m_Parameters) { std::string key(pair.first); @@ -169,8 +139,8 @@ void CampaignReader::InitParameters() if (key == "verbose") { - m_Verbosity = std::stoi(value); - if (m_Verbosity < 0 || m_Verbosity > 5) + m_Options.verbose = std::stoi(value); + if (m_Options.verbose < 0 || m_Options.verbose > 5) helper::Throw("Engine", "CampaignReader", "InitParameters", "Method verbose argument must be an " "integer in the range [0,5], in call to " @@ -178,37 +148,59 @@ void CampaignReader::InitParameters() } if (key == "hostname") { - m_Hostname = pair.second; + m_Options.hostname = pair.second; + } + if (key == "campaignstorepath") + { + m_Options.campaignstorepath = pair.second; } if (key == "cachepath") { - m_CachePath = pair.second; + m_Options.cachepath = pair.second; } } - if (m_Hostname.empty()) + if (m_Options.hostname.empty()) { - m_Hostname = helper::GetClusterName(); + m_Options.hostname = helper::GetClusterName(); + } + + if (m_Options.verbose > 0) + { + std::cout << "CampaignReader: \n"; + std::cout << " Hostname = " << m_Options.hostname << std::endl; + std::cout << " Campaign Store Path = " << m_Options.campaignstorepath << std::endl; + std::cout << " Cache Path = " << m_Options.cachepath << std::endl; } - // std::cout << "My Hostname is " << m_Hostname << std::endl; } void CampaignReader::InitTransports() { - int rc = sqlite3_open(m_Name.c_str(), &m_DB); + std::string path = m_Name; + if (!adios2sys::SystemTools::FileExists(path) && path[0] != '/' && path[0] != '\\' && + !m_Options.campaignstorepath.empty()) + { + std::string path2 = m_Options.campaignstorepath + PathSeparator + m_Name; + if (adios2sys::SystemTools::FileExists(path2)) + { + path = path2; + } + } + + int rc = sqlite3_open(path.c_str(), &m_DB); if (rc) { std::string dbmsg(sqlite3_errmsg(m_DB)); sqlite3_close(m_DB); helper::Throw("Engine", "CampaignReader", "Open", - "Cannot open database" + m_Name + ": " + dbmsg); + "Cannot open database" + path + ": " + dbmsg); } ReadCampaignData(m_DB, m_CampaignData); - if (m_Verbosity == 1) + if (m_Options.verbose == 1) { - std::cout << "Local hostname = " << m_Hostname << "\n"; + std::cout << "Local hostname = " << m_Options.hostname << "\n"; std::cout << "Database result:\n version = " << m_CampaignData.version << "\n hosts:\n"; for (size_t hostidx = 0; hostidx < m_CampaignData.hosts.size(); ++hostidx) @@ -243,18 +235,18 @@ void CampaignReader::InitTransports() { adios2::core::IO &io = m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); std::string localPath; - if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Hostname) + if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Options.hostname) { const std::string remotePath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; const std::string remoteURL = m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath; - if (m_Verbosity == 1) + if (m_Options.verbose == 1) { std::cout << "Open remote file " << remoteURL << "\n"; } - localPath = m_CachePath + PathSeparator + m_CampaignData.hosts[ds.hostIdx].hostname + - PathSeparator + ds.name; + localPath = m_Options.cachepath + PathSeparator + + m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + ds.name; helper::CreateDirectory(localPath); for (auto &bpf : ds.files) { @@ -269,7 +261,7 @@ void CampaignReader::InitTransports() { localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; - if (m_Verbosity == 1) + if (m_Options.verbose == 1) { std::cout << "Open local file " << localPath << "\n"; } @@ -311,7 +303,7 @@ void CampaignReader::InitTransports() void CampaignReader::DoClose(const int transportIndex) { - if (m_Verbosity == 5) + if (m_Options.verbose == 5) { std::cout << "Campaign Reader " << m_ReaderRank << " Close(" << m_Name << ")\n"; } diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 3678318f16..4747f7d875 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -54,10 +54,8 @@ class CampaignReader : public Engine bool VariableMinMax(const VariableBase &, const size_t Step, MinMaxStruct &MinMax); private: - int m_Verbosity = 0; // runtime parameter Verbose - std::string m_Hostname; // runtime parameter Hostname - std::string m_CachePath = "/tmp"; // runtime parameter CachePath - int m_ReaderRank; // my rank in the readers' comm + UserOptions::Campaign m_Options; + int m_ReaderRank; // my rank in the readers' comm int m_CurrentStep = 0; diff --git a/source/adios2/engine/sst/SstParamParser.cpp b/source/adios2/engine/sst/SstParamParser.cpp index ee8bd9a8cc..2b7041afa6 100644 --- a/source/adios2/engine/sst/SstParamParser.cpp +++ b/source/adios2/engine/sst/SstParamParser.cpp @@ -9,7 +9,8 @@ using namespace adios2; using namespace adios2::core; -void SstParamParser::ParseParams(IO &io, struct _SstParams &Params) +void SstParamParser::ParseParams(IO &io, struct _SstParams &Params, + const adios2::UserOptions &userOptions) { std::memset(&Params, 0, sizeof(Params)); @@ -254,6 +255,8 @@ void SstParamParser::ParseParams(IO &io, struct _SstParams &Params) return false; }; + Params.verbose = userOptions.sst.verbose; + #define get_params(Param, Type, Typedecl, Default) \ Params.Param = Default; \ lf_Set##Type##Parameter(#Param, Params.Param); diff --git a/source/adios2/engine/sst/SstParamParser.h b/source/adios2/engine/sst/SstParamParser.h index 0502cfcd5d..251c74cfa9 100644 --- a/source/adios2/engine/sst/SstParamParser.h +++ b/source/adios2/engine/sst/SstParamParser.h @@ -2,6 +2,7 @@ #define ADIOS2_ENGINE_SST_SSTPARAMPARSER_H_ #include "adios2/common/ADIOSConfig.h" +#include "adios2/common/ADIOSTypes.h" #include "adios2/core/IO.h" #include #include @@ -13,7 +14,8 @@ using namespace adios2::core; class SstParamParser { public: - void ParseParams(adios2::core::IO &io, struct _SstParams &Params); + void ParseParams(adios2::core::IO &io, struct _SstParams &Params, + const adios2::UserOptions &userOptions); }; #endif diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp index 2754e75f6e..03bfced452 100644 --- a/source/adios2/engine/sst/SstReader.cpp +++ b/source/adios2/engine/sst/SstReader.cpp @@ -509,7 +509,7 @@ void SstReader::Init() { SstParamParser Parser; - Parser.ParseParams(m_IO, Params); + Parser.ParseParams(m_IO, Params, m_UserOptions); } #define declare_gets(T) \ diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp index 305578ef5c..146ff0f735 100644 --- a/source/adios2/engine/sst/SstWriter.cpp +++ b/source/adios2/engine/sst/SstWriter.cpp @@ -389,7 +389,7 @@ void SstWriter::Init() { SstParamParser Parser; - Parser.ParseParams(m_IO, Params); + Parser.ParseParams(m_IO, Params, m_UserOptions); if (Params.verbose < 0 || Params.verbose > 5) { diff --git a/source/adios2/helper/adiosUserOptions.cpp b/source/adios2/helper/adiosUserOptions.cpp new file mode 100644 index 0000000000..34db1778ff --- /dev/null +++ b/source/adios2/helper/adiosUserOptions.cpp @@ -0,0 +1,121 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + */ + +#include "adiosUserOptions.h" + +#include "adios2/helper/adiosString.h" + +#include + +namespace adios2 +{ +namespace helper +{ + +namespace +{ + +inline void FixHomePath(std::string &path, std::string &homePath) +{ + if (!path.empty() && path[0] == '~') + { + path = homePath + path.substr(1); + } +} + +constexpr bool isMandatory = true; +constexpr bool isNotMandatory = false; + +YAML::Node YAMLNode(const std::string nodeName, const YAML::Node &upperNode, + const std::string &hint, const bool isMandatory, + const YAML::NodeType::value nodeType) +{ + const YAML::Node node = upperNode[nodeName]; + + if (isMandatory && !node) + { + helper::Throw( + "Helper", "adiosYAML", "YAMLNode", + "no " + nodeName + " node found, (is your node key lower case?), " + hint); + } + if (node && node.Type() != nodeType) + { + helper::Throw("Helper", "adiosYAML", "YAMLNode", + "node " + nodeName + + " is the wrong type, review adios2 " + "config YAML specs for the node, " + + hint); + } + return node; +} + +} // end empty namespace + +void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, + std::string &homePath) +{ + const std::string hint = + "when parsing user config file " + configFileYAML + " in call to ADIOS constructor"; + + const std::string configFileContents = comm.BroadcastFile(configFileYAML, hint); + const YAML::Node document = YAML::Load(configFileContents); + + if (!document) + { + helper::Throw( + "Helper", "adiosUserOptions", "ParseUserOptionsFile", + "parser error in file " + configFileYAML + + " invalid format. Check with any YAML editor if format is ill-formed, " + hint); + } + + /* + * This code section below determines what options we recognize at all from the + * ~/.config/adios2/adios2.yaml file + */ + + for (auto itNode = document.begin(); itNode != document.end(); ++itNode) + { + std::cout << itNode->const YAML::Node general = + YAMLNode("IO", *itNode, hint, isNotMandatory, YAML::NodeType::Scalar); + if (ioScalar) + { + const std::string ioName = ioScalar.as(); + // Build the IO object + auto itCurrentIO = + ios.emplace(std::piecewise_construct, std::forward_as_tuple(ioName), + std::forward_as_tuple(adios, ioName, true, adios.m_HostLanguage)); + core::IO ¤tIO = itCurrentIO.first->second; + IOYAML(adios, *itNode, currentIO, hint); + } + } + + /* + if (iniReader.HasSection("General")) + { + options.general.verbose = static_cast(iniReader.GetInteger("General", "verbose", + 0)); + } + + if (iniReader.HasSection("Campaign")) + { + options.campaign.active = iniReader.GetBoolean("Campaign", "active", true); + options.campaign.verbose = static_cast(iniReader.GetInteger("Campaign", "verbose", + 0)); options.campaign.hostname = iniReader.Get("Campaign", "hostname", ""); + options.campaign.campaignstorepath = iniReader.Get("Campaign", "campaignstorepath", ""); + FixHomePath(options.campaign.campaignstorepath, homePath); + options.campaign.cachepath = iniReader.Get("Campaign", "cachepath", + "/tmp/adios2-cache"); FixHomePath(options.campaign.cachepath, homePath); + } + + if (iniReader.HasSection("SST")) + { + options.sst.verbose = static_cast(iniReader.GetInteger("SST", "verbose", 0)); + } + */ +} + +} // end namespace helper +} // end namespace adios2 diff --git a/source/adios2/helper/adiosUserOptions.h b/source/adios2/helper/adiosUserOptions.h new file mode 100644 index 0000000000..aec207f076 --- /dev/null +++ b/source/adios2/helper/adiosUserOptions.h @@ -0,0 +1,30 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * adiosUserOptions.h YAML file parsing functionality for ~/.config/adios2/adios2.yaml + */ + +#ifndef ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ +#define ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ + +/// \cond EXCLUDE_FROM_DOXYGEN +#include +#include +/// \endcond + +#include "adios2/common/ADIOSTypes.h" +#include "adios2/helper/adiosComm.h" + +namespace adios2 +{ +namespace helper +{ + +void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, + std::string &homePath); + +} // end namespace helper +} // end namespace adios2 + +#endif /* ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ */ diff --git a/source/adios2/helper/adiosYAML.cpp b/source/adios2/helper/adiosYAML.cpp index ec835ce26c..4283e8b676 100644 --- a/source/adios2/helper/adiosYAML.cpp +++ b/source/adios2/helper/adiosYAML.cpp @@ -65,6 +65,44 @@ Params YAMLNodeMapToParams(const YAML::Node &node, const std::string &hint) constexpr bool isMandatory = true; constexpr bool isNotMandatory = false; + +inline void FixHomePath(std::string &path, std::string &homePath) +{ + if (!path.empty() && path[0] == '~') + { + path = homePath + path.substr(1); + } +} + +/*std::string NodeType(const YAML::Node &node) +{ + switch (node.Type()) + { + case YAML::NodeType::Null: + return "Null"; + case YAML::NodeType::Scalar: + return "Scalar"; + case YAML::NodeType::Sequence: + return "Sequence"; + case YAML::NodeType::Map: + return "Map"; + case YAML::NodeType::Undefined: + return "Undefined"; + } + return "NoIdeaWhatThisIs"; +}*/ + +template +void SetOption(T &value, const std::string nodeName, const YAML::Node &upperNode, + const std::string &hint) +{ + auto node = YAMLNode(nodeName, upperNode, hint, isNotMandatory, YAML::NodeType::Scalar); + if (node) + { + value = node.as(); + } +} + } // end empty namespace void IOVariableYAML(const YAML::Node &variableMap, core::IO ¤tIO, const std::string &hint) @@ -220,5 +258,75 @@ std::string ParseConfigYAML(core::ADIOS &adios, const std::string &configFileYAM return configFileContents; } +void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, + std::string &homePath) +{ + const std::string hint = + "when parsing user config file " + configFileYAML + " in call to ADIOS constructor"; + + const std::string configFileContents = comm.BroadcastFile(configFileYAML, hint); + + /* + * Set defaults first + */ + options.general.verbose = 0; + + options.campaign.active = true; + options.campaign.verbose = 0; + options.campaign.hostname = ""; + options.campaign.campaignstorepath = ""; + options.campaign.cachepath = "/tmp/adios2-cache"; + + options.sst.verbose = 0; + + const YAML::Node document = YAML::Load(configFileContents); + if (!document) + { + helper::Throw( + "Helper", "adiosUserOptions", "ParseUserOptionsFile", + "parser error in file " + configFileYAML + + " invalid format. Check with any YAML editor if format is ill-formed, " + hint); + } + + /* + * This code section below determines what options we recognize at all from the + * ~/.config/adios2/adios2.yaml file + */ + { + UserOptions::General &opts = options.general; + const YAML::Node general = + YAMLNode("General", document, hint, isNotMandatory, YAML::NodeType::Map); + if (general) + { + SetOption(opts.verbose, "verbose", general, hint); + } + } + + { + UserOptions::Campaign &opts = options.campaign; + const YAML::Node campaign = + YAMLNode("Campaign", document, hint, isNotMandatory, YAML::NodeType::Map); + if (campaign) + { + SetOption(opts.verbose, "verbose", campaign, hint); + SetOption(opts.active, "active", campaign, hint); + SetOption(opts.hostname, "hostname", campaign, hint); + SetOption(opts.campaignstorepath, "campaignstorepath", campaign, hint); + FixHomePath(opts.campaignstorepath, homePath); + SetOption(opts.cachepath, "cachepath", campaign, hint); + FixHomePath(opts.cachepath, homePath); + } + } + + { + UserOptions::SST &opts = options.sst; + const YAML::Node sst = YAMLNode("SST", document, hint, isNotMandatory, YAML::NodeType::Map); + if (sst) + { + SetOption(opts.verbose, "verbose", sst, hint); + } + } +} + } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosYAML.h b/source/adios2/helper/adiosYAML.h index 8014d0cf06..8bb9df19d5 100644 --- a/source/adios2/helper/adiosYAML.h +++ b/source/adios2/helper/adiosYAML.h @@ -18,6 +18,7 @@ #include //std::pair /// \endcond +#include "adios2/common/ADIOSTypes.h" // UserOptions #include "adios2/core/ADIOS.h" #include "adios2/core/IO.h" @@ -31,6 +32,9 @@ void ParseConfigYAMLIO(core::ADIOS &adios, const std::string &configFileYAML, std::string ParseConfigYAML(core::ADIOS &adios, const std::string &configFileYAML, std::map &ios); +void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, + std::string &homePath); + } // end namespace helper } // end namespace adios2 diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index 0d3ce8e73c..52e8e01b2b 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -1555,13 +1555,25 @@ std::vector getEnginesList(const std::string path) return list; } -int doList(const char *path) +int doList(std::string path) { char init_params[128]; int adios_verbose = 2; if (verbose > 1) - printf("\nADIOS Open: read header info from %s\n", path); + printf("\nADIOS Open: read header info from %s\n", path.c_str()); + + // initialize BP reader + if (verbose > 1) + adios_verbose = 3; // print info lines + if (verbose > 2) + adios_verbose = 4; // print debug lines + snprintf(init_params, sizeof(init_params), "verbose=%d", adios_verbose); + if (hidden_attrs) + strcat(init_params, ";show_hidden_attrs"); + + core::ADIOS adios("C++"); + const adios2::UserOptions userOptions = adios.GetUserOptions(); std::string tpl = helper::LowerCase(transport_params); bool remoteFile = @@ -1579,23 +1591,34 @@ int doList(const char *path) } else { - if (!adios2sys::SystemTools::FileExists(path)) + bool exists = adios2sys::SystemTools::FileExists(path); + if (!exists && !userOptions.campaign.campaignstorepath.empty() && path[0] != PathSeparator) { - fprintf(stderr, "\nError: input path %s does not exist\n", path); + std::string path2 = userOptions.campaign.campaignstorepath + PathSeparator + path; + exists = adios2sys::SystemTools::FileExists(path2); + if (exists) + { + path = path2.c_str(); + } + else + { + std::string path3 = + userOptions.campaign.campaignstorepath + PathSeparator + path + ".aca"; + exists = adios2sys::SystemTools::FileExists(path3); + if (exists) + { + path = path3.c_str(); + } + } + } + + if (!exists) + { + fprintf(stderr, "\nError: input path %s does not exist\n", path.c_str()); return 4; } } - // initialize BP reader - if (verbose > 1) - adios_verbose = 3; // print info lines - if (verbose > 2) - adios_verbose = 4; // print debug lines - snprintf(init_params, sizeof(init_params), "verbose=%d", adios_verbose); - if (hidden_attrs) - strcat(init_params, ";show_hidden_attrs"); - - core::ADIOS adios("C++"); core::IO &io = adios.DeclareIO("bpls"); if (timestep) { diff --git a/source/utils/bpls/bpls.h b/source/utils/bpls/bpls.h index 683c582b4d..8d5c9be035 100644 --- a/source/utils/bpls/bpls.h +++ b/source/utils/bpls/bpls.h @@ -17,6 +17,7 @@ #include "adios2/core/Info.h" #include +#include namespace adios2 { @@ -60,7 +61,7 @@ void parseDimSpec(const std::string &str, int64_t *dims); int parseAccuracy(); int compile_regexp_masks(void); void printSettings(void); -int doList(const char *path); +int doList(std::string path); void mergeLists(int nV, char **listV, int nA, char **listA, char **mlist, bool *isVar); template From 1ee9c00d18e259917cd7f9424e48dfa9d573e814 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 09:57:18 -0400 Subject: [PATCH 02/11] Fix compiler error Remove extra file not needed --- .../adios2/engine/campaign/CampaignManager.h | 2 +- source/adios2/helper/adiosUserOptions.cpp | 121 ------------------ source/adios2/helper/adiosUserOptions.h | 30 ----- 3 files changed, 1 insertion(+), 152 deletions(-) delete mode 100644 source/adios2/helper/adiosUserOptions.cpp delete mode 100644 source/adios2/helper/adiosUserOptions.h diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h index 498de83caa..2a23064cd6 100644 --- a/source/adios2/engine/campaign/CampaignManager.h +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -57,7 +57,7 @@ class CampaignManager public: CampaignManager(helper::Comm &comm){}; ~CampaignManager() = default; - void Open(const std::string &name, const INIOptions &options){}; + void Open(const std::string &name, const UserOptions &options){}; void Record(const std::string &name, const size_t step, const double time){}; void Close(){}; diff --git a/source/adios2/helper/adiosUserOptions.cpp b/source/adios2/helper/adiosUserOptions.cpp deleted file mode 100644 index 34db1778ff..0000000000 --- a/source/adios2/helper/adiosUserOptions.cpp +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - */ - -#include "adiosUserOptions.h" - -#include "adios2/helper/adiosString.h" - -#include - -namespace adios2 -{ -namespace helper -{ - -namespace -{ - -inline void FixHomePath(std::string &path, std::string &homePath) -{ - if (!path.empty() && path[0] == '~') - { - path = homePath + path.substr(1); - } -} - -constexpr bool isMandatory = true; -constexpr bool isNotMandatory = false; - -YAML::Node YAMLNode(const std::string nodeName, const YAML::Node &upperNode, - const std::string &hint, const bool isMandatory, - const YAML::NodeType::value nodeType) -{ - const YAML::Node node = upperNode[nodeName]; - - if (isMandatory && !node) - { - helper::Throw( - "Helper", "adiosYAML", "YAMLNode", - "no " + nodeName + " node found, (is your node key lower case?), " + hint); - } - if (node && node.Type() != nodeType) - { - helper::Throw("Helper", "adiosYAML", "YAMLNode", - "node " + nodeName + - " is the wrong type, review adios2 " - "config YAML specs for the node, " + - hint); - } - return node; -} - -} // end empty namespace - -void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, - std::string &homePath) -{ - const std::string hint = - "when parsing user config file " + configFileYAML + " in call to ADIOS constructor"; - - const std::string configFileContents = comm.BroadcastFile(configFileYAML, hint); - const YAML::Node document = YAML::Load(configFileContents); - - if (!document) - { - helper::Throw( - "Helper", "adiosUserOptions", "ParseUserOptionsFile", - "parser error in file " + configFileYAML + - " invalid format. Check with any YAML editor if format is ill-formed, " + hint); - } - - /* - * This code section below determines what options we recognize at all from the - * ~/.config/adios2/adios2.yaml file - */ - - for (auto itNode = document.begin(); itNode != document.end(); ++itNode) - { - std::cout << itNode->const YAML::Node general = - YAMLNode("IO", *itNode, hint, isNotMandatory, YAML::NodeType::Scalar); - if (ioScalar) - { - const std::string ioName = ioScalar.as(); - // Build the IO object - auto itCurrentIO = - ios.emplace(std::piecewise_construct, std::forward_as_tuple(ioName), - std::forward_as_tuple(adios, ioName, true, adios.m_HostLanguage)); - core::IO ¤tIO = itCurrentIO.first->second; - IOYAML(adios, *itNode, currentIO, hint); - } - } - - /* - if (iniReader.HasSection("General")) - { - options.general.verbose = static_cast(iniReader.GetInteger("General", "verbose", - 0)); - } - - if (iniReader.HasSection("Campaign")) - { - options.campaign.active = iniReader.GetBoolean("Campaign", "active", true); - options.campaign.verbose = static_cast(iniReader.GetInteger("Campaign", "verbose", - 0)); options.campaign.hostname = iniReader.Get("Campaign", "hostname", ""); - options.campaign.campaignstorepath = iniReader.Get("Campaign", "campaignstorepath", ""); - FixHomePath(options.campaign.campaignstorepath, homePath); - options.campaign.cachepath = iniReader.Get("Campaign", "cachepath", - "/tmp/adios2-cache"); FixHomePath(options.campaign.cachepath, homePath); - } - - if (iniReader.HasSection("SST")) - { - options.sst.verbose = static_cast(iniReader.GetInteger("SST", "verbose", 0)); - } - */ -} - -} // end namespace helper -} // end namespace adios2 diff --git a/source/adios2/helper/adiosUserOptions.h b/source/adios2/helper/adiosUserOptions.h deleted file mode 100644 index aec207f076..0000000000 --- a/source/adios2/helper/adiosUserOptions.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * adiosUserOptions.h YAML file parsing functionality for ~/.config/adios2/adios2.yaml - */ - -#ifndef ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ -#define ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ - -/// \cond EXCLUDE_FROM_DOXYGEN -#include -#include -/// \endcond - -#include "adios2/common/ADIOSTypes.h" -#include "adios2/helper/adiosComm.h" - -namespace adios2 -{ -namespace helper -{ - -void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, - std::string &homePath); - -} // end namespace helper -} // end namespace adios2 - -#endif /* ADIOS2_HELPER_ADIOSUSEROPTIONS_H_ */ From ab83bbb65420f35ef32066c2738c516ff64c9e69 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 10:48:33 -0400 Subject: [PATCH 03/11] do not include unistd.h --- source/adios2/engine/campaign/CampaignData.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index fa6dda7770..9a6cefc8a1 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include namespace adios2 From 54f3d5e2ba198bb9621e384f67f5e87abe8b57a2 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 11:21:32 -0400 Subject: [PATCH 04/11] change a long variable to int64_t to avoid size confusion on windows --- source/adios2/engine/campaign/CampaignData.cpp | 17 ++++++++--------- source/adios2/engine/campaign/CampaignData.h | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index 9a6cefc8a1..edce699fb6 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -91,8 +91,7 @@ static int sqlcb_bpfile(void *p, int argc, char **argv, char **azColName) helper::StringToSizeT(std::string(argv[3]), "SQL callback convert text to int"); cf.lengthCompressed = helper::StringToSizeT(std::string(argv[4]), "SQL callback convert text to int"); - cf.ctime = static_cast( - helper::StringTo(std::string(argv[5]), "SQL callback convert ctime to int")); + cf.ctime = helper::StringTo(std::string(argv[5]), "SQL callback convert ctime to int"); CampaignBPDataset &cds = cdp->bpdatasets[cf.bpDatasetIdx]; cds.files.push_back(cf); @@ -232,9 +231,9 @@ int inflateToFile(const unsigned char *source, const size_t blobsize, std::ofstr return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; } -static long timeToSec(long ct) +static int64_t timeToSec(int64_t ct) { - long t; + int64_t t; if (ct > 99999999999999999) { /* nanosec to sec */ @@ -257,7 +256,7 @@ static long timeToSec(long ct) return t; } -static bool isFileNewer(const std::string path, long ctime) +static bool isFileNewer(const std::string path, int64_t ctime) { int result; #ifdef _WIN32 @@ -272,9 +271,9 @@ static bool isFileNewer(const std::string path, long ctime) return false; } - long ct = s.st_ctime; - long ctSec = timeToSec(ct); - long ctimeSec = timeToSec(ctime); + int64_t ct = static_cast(s.st_ctime); + int64_t ctSec = timeToSec(ct); + int64_t ctimeSec = timeToSec(ctime); /*std::cout << " Stat(" << path << "): size = " << s.st_size << " ct = " << ctSec << " ctime = " << ctimeSec << "\n";*/ @@ -297,7 +296,7 @@ void SaveToFile(sqlite3 *db, const std::string &path, const CampaignBPFile &bpfi sqlcmd = "SELECT data FROM bpfile WHERE bpdatasetid = " + id + " AND name = '" + bpfile.name + "'"; // std::cout << "SQL statement: " << sqlcmd << "\n"; - rc = sqlite3_prepare_v2(db, sqlcmd.c_str(), sqlcmd.size(), &statement, NULL); + rc = sqlite3_prepare_v2(db, sqlcmd.c_str(), static_cast(sqlcmd.size()), &statement, NULL); if (rc != SQLITE_OK) { std::cout << "SQL error: " << zErrMsg << std::endl; diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index 8d360abfa4..d177ad30a6 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -41,7 +41,7 @@ struct CampaignBPFile bool compressed; size_t lengthOriginal; size_t lengthCompressed; - long ctime; + int64_t ctime; }; struct CampaignBPDataset From 4904f91a7a32a9d8144f22433593a12fab57365c Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 13:00:38 -0400 Subject: [PATCH 05/11] Use yaml parser in campaign manager python script --- .../adios2_campaign_manager.py | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 171af0b4cc..88036afdd4 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -4,6 +4,8 @@ import glob import sqlite3 import zlib +import yaml +from dataclasses import dataclass from datetime import datetime from os import chdir, getcwd, remove, stat from os.path import exists, isdir, expanduser @@ -15,19 +17,32 @@ ADIOS_ACA_VERSION = "1.0" +@dataclass +class UserOption: + adios_campaign_store: str = None + hostname: str = None + verbose: int = 0 -def ReadConfig(): - path = expanduser("~/.config/adios2/campaign.cfg") + +def ReadUserConfig(): + path = expanduser("~/.config/adios2/adios2.yaml") + opts = UserOption() try: + doc = {} with open(path) as f: - lines = f.readlines() - for line in lines: - lst = line.split() - if lst[0] == "campaignstorepath": - adios_campaign_store = expanduser(lst[1]) + doc = yaml.safe_load(f) + camp = doc.get("Campaign") + if isinstance(camp, dict): + for key, value in camp.items(): + if key == "campaignstorepath": + opts.adios_campaign_store = expanduser(value) + if key == "hostname": + opts.hostname = value + if key == "verbose": + opts.verbose = value except FileNotFoundError: - adios_campaign_store = None - return adios_campaign_store + None + return opts def SetupArgs(): @@ -50,9 +65,13 @@ def SetupArgs(): args = parser.parse_args() # default values - args.update = False + args.user_options = ReadUserConfig() + + if args.verbose == 0: + args.verbose = args.user_options.verbose + if args.campaign_store is None: - args.campaign_store = ReadConfig() + args.campaign_store = args.user_options.adios_campaign_store if args.campaign_store is not None: while args.campaign_store[-1] == "/": @@ -223,7 +242,10 @@ def GetHostName(): host = sub("^login[0-9]*\\.", "", host) if host.startswith("batch"): host = sub("^batch[0-9]*\\.", "", host) - shorthost = host.split(".")[0] + if args.user_options.hostname is None: + shorthost = host.split(".")[0] + else: + shorthost = args.user_options.hostname return host, shorthost @@ -278,8 +300,6 @@ def AddDirectory(hostID: int, path: str) -> int: def Update(args: dict, cur: sqlite3.Cursor): longHostName, shortHostName = GetHostName() - if args.hostname is not None: - shortHostName = args.hostname hostID = AddHostName(longHostName, shortHostName) @@ -355,6 +375,7 @@ def Info(args: dict, cur: sqlite3.Cursor): '"' ) bpdatasets = res3.fetchall() + print(f"BPDATASETS type = {type(bpdatasets)}") for bpdataset in bpdatasets: t = timestamp_to_datetime(bpdataset[2]) print(f" dataset = {bpdataset[1]} created on {t}") From d3251efc4e23c025861ac746a6e2d564903ab091 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 13:01:02 -0400 Subject: [PATCH 06/11] bug fix: the order of entries in bpdataset table is undefined but the campaign data reader relied on calculating the index as if it was sorted by the insertion order. Use a map instead to store the rowid and use that as index for the bpfile elements. --- source/adios2/engine/campaign/CampaignData.cpp | 15 ++++++++------- source/adios2/engine/campaign/CampaignData.h | 6 +++--- source/adios2/engine/campaign/CampaignReader.cpp | 6 ++++-- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index edce699fb6..1f003052a6 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -69,12 +69,13 @@ static int sqlcb_bpdataset(void *p, int argc, char **argv, char **azColName) { CampaignData *cdp = reinterpret_cast(p); CampaignBPDataset cds; - size_t hostid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); - size_t dirid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int"); + size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); + size_t hostid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int"); + size_t dirid = helper::StringToSizeT(std::string(argv[2]), "SQL callback convert text to int"); cds.hostIdx = hostid - 1; // SQL rows start from 1, vector idx start from 0 cds.dirIdx = dirid - 1; - cds.name = argv[2]; - cdp->bpdatasets.push_back(cds); + cds.name = argv[3]; + cdp->bpdatasets[dsid] = cds; return 0; }; @@ -83,7 +84,7 @@ static int sqlcb_bpfile(void *p, int argc, char **argv, char **azColName) CampaignData *cdp = reinterpret_cast(p); CampaignBPFile cf; size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); - cf.bpDatasetIdx = dsid - 1; + cf.bpDatasetIdx = dsid; cf.name = std::string(argv[1]); int comp = helper::StringTo(std::string(argv[2]), "SQL callback convert text to int"); cf.compressed = (bool)comp; @@ -137,7 +138,7 @@ void ReadCampaignData(sqlite3 *db, CampaignData &cd) sqlite3_free(zErrMsg); } - sqlcmd = "SELECT hostid, dirid, name FROM bpdataset"; + sqlcmd = "SELECT rowid, hostid, dirid, name FROM bpdataset"; rc = sqlite3_exec(db, sqlcmd.c_str(), sqlcb_bpdataset, &cd, &zErrMsg); if (rc != SQLITE_OK) { @@ -290,7 +291,7 @@ void SaveToFile(sqlite3 *db, const std::string &path, const CampaignBPFile &bpfi int rc; char *zErrMsg = 0; std::string sqlcmd; - std::string id = std::to_string(bpfile.bpDatasetIdx + 1); + std::string id = std::to_string(bpfile.bpDatasetIdx); sqlite3_stmt *statement; sqlcmd = diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index d177ad30a6..140cfb89cf 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -14,8 +14,8 @@ #include #include +#include #include -#include #include #include @@ -37,7 +37,7 @@ struct CampaignHost struct CampaignBPFile { std::string name; - size_t bpDatasetIdx; + size_t bpDatasetIdx; // index of parent CampaignBPDataset in the map bool compressed; size_t lengthOriginal; size_t lengthCompressed; @@ -56,7 +56,7 @@ struct CampaignData { std::string version; std::vector hosts; - std::vector bpdatasets; + std::map bpdatasets; }; void ReadCampaignData(sqlite3 *db, CampaignData &cd); diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 2bec32bf22..79cf87ba22 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -214,8 +214,9 @@ void CampaignReader::InitTransports() } } std::cout << " datasets:\n"; - for (auto &ds : m_CampaignData.bpdatasets) + for (auto &it : m_CampaignData.bpdatasets) { + CampaignBPDataset &ds = it.second; std::cout << " " << m_CampaignData.hosts[ds.hostIdx].hostname << ":" << m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] << PathSeparator << ds.name << "\n"; @@ -231,8 +232,9 @@ void CampaignReader::InitTransports() // std::cout << "JSON rank " << m_ReaderRank << ": " << js.size() << // std::endl; int i = 0; - for (auto &ds : m_CampaignData.bpdatasets) + for (auto &it : m_CampaignData.bpdatasets) { + CampaignBPDataset &ds = it.second; adios2::core::IO &io = m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); std::string localPath; if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Options.hostname) From f0b643606fbe77c1c02fab00256076b11013e569 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 13:16:26 -0400 Subject: [PATCH 07/11] rename remote_server to adios2_remote_server --- source/adios2/toolkit/remote/CMakeLists.txt | 10 +++++----- source/adios2/toolkit/remote/remote_server.cpp | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/adios2/toolkit/remote/CMakeLists.txt b/source/adios2/toolkit/remote/CMakeLists.txt index 9b613d4b45..a739e1ad63 100644 --- a/source/adios2/toolkit/remote/CMakeLists.txt +++ b/source/adios2/toolkit/remote/CMakeLists.txt @@ -4,9 +4,9 @@ #------------------------------------------------------------------------------# if (NOT ADIOS2_USE_PIP) - add_executable(remote_server ./remote_server.cpp remote_common.cpp) + add_executable(adios2_remote_server ./remote_server.cpp remote_common.cpp) - target_link_libraries(remote_server PUBLIC EVPath::EVPath adios2_core adios2sys + target_link_libraries(adios2_remote_server PUBLIC EVPath::EVPath adios2_core adios2sys PRIVATE $<$:shlwapi>) get_property(pugixml_headers_path @@ -14,10 +14,10 @@ if (NOT ADIOS2_USE_PIP) PROPERTY INTERFACE_INCLUDE_DIRECTORIES ) - target_include_directories(remote_server PRIVATE ${PROJECT_BINARY_DIR} ${pugixml_headers_path}) + target_include_directories(adios2_remote_server PRIVATE ${PROJECT_BINARY_DIR} ${pugixml_headers_path}) - set_property(TARGET remote_server PROPERTY OUTPUT_NAME remote_server${ADIOS2_EXECUTABLE_SUFFIX}) - install(TARGETS remote_server EXPORT adios2 + set_property(TARGET adios2_remote_server PROPERTY OUTPUT_NAME adios2_remote_server${ADIOS2_EXECUTABLE_SUFFIX}) + install(TARGETS adios2_remote_server EXPORT adios2 RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime ) endif () diff --git a/source/adios2/toolkit/remote/remote_server.cpp b/source/adios2/toolkit/remote/remote_server.cpp index e46e06bca0..1dedea3d09 100644 --- a/source/adios2/toolkit/remote/remote_server.cpp +++ b/source/adios2/toolkit/remote/remote_server.cpp @@ -543,7 +543,7 @@ int main(int argc, char **argv) else { fprintf(stderr, "Unknown argument \"%s\"\n", argv[i]); - fprintf(stderr, "Usage: remote_server [-background] [-kill_server] [-no_timeout] " + fprintf(stderr, "Usage: adios2_remote_server [-background] [-kill_server] [-no_timeout] " "[-status] [-v] [-q]\n"); exit(1); } From 76af386b3a6ca377dd4c314cea9837352323ddd5 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 13:33:09 -0400 Subject: [PATCH 08/11] Use the name of the campaign in the cache path to avoid name collision --- .../adios2/engine/campaign/CampaignReader.cpp | 28 ++++++++++--------- source/utils/bpls/bpls.cpp | 4 +-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 79cf87ba22..a8c0cc0fde 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -33,19 +33,19 @@ CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, : Engine("CampaignReader", io, name, mode, std::move(comm)) { m_ReaderRank = m_Comm.Rank(); - Init(); - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " Open(" << m_Name << ") in constructor." << std::endl; } + Init(); m_IsOpen = true; } CampaignReader::~CampaignReader() { /* CampaignReader destructor does close and finalize */ - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " destructor on " << m_Name << "\n"; } @@ -62,7 +62,7 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, const float timeoutSec // so this forced increase should not be here ++m_CurrentStep; - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " BeginStep() new step " << m_CurrentStep << "\n"; @@ -89,7 +89,7 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, const float timeoutSec void CampaignReader::PerformGets() { - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " PerformGets()\n"; } @@ -107,7 +107,7 @@ void CampaignReader::EndStep() PerformGets(); } - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " EndStep()\n"; } @@ -198,7 +198,7 @@ void CampaignReader::InitTransports() ReadCampaignData(m_DB, m_CampaignData); - if (m_Options.verbose == 1) + if (m_Options.verbose > 0) { std::cout << "Local hostname = " << m_Options.hostname << "\n"; std::cout << "Database result:\n version = " << m_CampaignData.version << "\n hosts:\n"; @@ -243,12 +243,14 @@ void CampaignReader::InitTransports() m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; const std::string remoteURL = m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath; - if (m_Options.verbose == 1) + localPath = m_Options.cachepath + PathSeparator + + m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + m_Name + + PathSeparator + ds.name; + if (m_Options.verbose > 0) { - std::cout << "Open remote file " << remoteURL << "\n"; + std::cout << "Open remote file " << remoteURL + << "\n and use local cache for metadata at " << localPath << " \n"; } - localPath = m_Options.cachepath + PathSeparator + - m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + ds.name; helper::CreateDirectory(localPath); for (auto &bpf : ds.files) { @@ -263,7 +265,7 @@ void CampaignReader::InitTransports() { localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; - if (m_Options.verbose == 1) + if (m_Options.verbose > 0) { std::cout << "Open local file " << localPath << "\n"; } @@ -305,7 +307,7 @@ void CampaignReader::InitTransports() void CampaignReader::DoClose(const int transportIndex) { - if (m_Options.verbose == 5) + if (m_Options.verbose > 1) { std::cout << "Campaign Reader " << m_ReaderRank << " Close(" << m_Name << ")\n"; } diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index 52e8e01b2b..94dd7805cf 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -1598,7 +1598,7 @@ int doList(std::string path) exists = adios2sys::SystemTools::FileExists(path2); if (exists) { - path = path2.c_str(); + ; // path = path2.c_str(); } else { @@ -1607,7 +1607,7 @@ int doList(std::string path) exists = adios2sys::SystemTools::FileExists(path3); if (exists) { - path = path3.c_str(); + path += ".aca"; // path = path3.c_str(); } } } From 98e9f761e17374ff30ffc10e612bd58ef58a7ac2 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 13:38:07 -0400 Subject: [PATCH 09/11] clang-format --- source/adios2/toolkit/remote/remote_server.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/adios2/toolkit/remote/remote_server.cpp b/source/adios2/toolkit/remote/remote_server.cpp index 1dedea3d09..5108ed421e 100644 --- a/source/adios2/toolkit/remote/remote_server.cpp +++ b/source/adios2/toolkit/remote/remote_server.cpp @@ -543,8 +543,9 @@ int main(int argc, char **argv) else { fprintf(stderr, "Unknown argument \"%s\"\n", argv[i]); - fprintf(stderr, "Usage: adios2_remote_server [-background] [-kill_server] [-no_timeout] " - "[-status] [-v] [-q]\n"); + fprintf(stderr, + "Usage: adios2_remote_server [-background] [-kill_server] [-no_timeout] " + "[-status] [-v] [-q]\n"); exit(1); } } From fafb640294bd5ec7680a450dba9b9c2ac345d059 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 16:42:24 -0400 Subject: [PATCH 10/11] fix remote server test, use the new binary name --- testing/adios2/engine/bp/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index 8c5b90606b..bf99106bfa 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -120,10 +120,10 @@ if ((NOT WIN32) AND ADIOS2_HAVE_SST) set_tests_properties(Remote.BP${testname}.FileRemote PROPERTIES FIXTURES_REQUIRED Server ENVIRONMENT "DoFileRemote=1") endmacro() - add_test(NAME remoteServerSetup COMMAND remote_server -background) + add_test(NAME remoteServerSetup COMMAND adios2_remote_server -background) set_tests_properties(remoteServerSetup PROPERTIES FIXTURES_SETUP Server) - add_test(NAME remoteServerCleanup COMMAND remote_server -kill_server) + add_test(NAME remoteServerCleanup COMMAND adios2_remote_server -kill_server) set_tests_properties(remoteServerCleanup PROPERTIES FIXTURES_CLEANUP Server) ##### add remote tests below this line From 839758756bf84c0d74a6c844e302d19c9c898f34 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sat, 30 Mar 2024 17:00:06 -0400 Subject: [PATCH 11/11] used size_t not int for map indexing to avoid type conversion --- source/adios2/engine/campaign/CampaignData.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index 140cfb89cf..fa316c1bcb 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -56,7 +56,7 @@ struct CampaignData { std::string version; std::vector hosts; - std::map bpdatasets; + std::map bpdatasets; }; void ReadCampaignData(sqlite3 *db, CampaignData &cd);