Skip to content

Commit

Permalink
Merge pull request #4119 from pnorbert/user-options-yaml
Browse files Browse the repository at this point in the history
Add support for user options in ~/.config/adios2/adios2.yaml
  • Loading branch information
pnorbert authored Apr 1, 2024
2 parents dc6c981 + 8397587 commit 56320ec
Show file tree
Hide file tree
Showing 24 changed files with 377 additions and 157 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ adios_option(Profiling "Enable support for profiling" AUTO)
adios_option(Endian_Reverse "Enable support for Little/Big Endian Interoperability" AUTO)
adios_option(Sodium "Enable support for Sodium for encryption" AUTO)
adios_option(Catalyst "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO)
adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB)" OFF)
adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB)" AUTO)
adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF)
adios_option(Derived_Variable "Enable support for derived variables" OFF)
adios_option(PIP "Enable support for pip packaging" OFF)
Expand Down
27 changes: 27 additions & 0 deletions source/adios2/common/ADIOSTypes.h
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,33 @@ std::string ToString(const Dims &dims);
std::string ToString(const Box<Dims> &box);
std::string ToString(const MemorySpace value);

/** UserOptions holds all user options from ~/.config/adios2/adios2.yaml */
struct UserOptions
{
struct General
{
int verbose;
};

struct Campaign
{
bool active;
int verbose;
std::string hostname;
std::string campaignstorepath;
std::string cachepath;
};

struct SST
{
int verbose;
};

General general;
Campaign campaign;
SST sst;
};

/**
* os << [adios2_type] enables output of adios2 enums/classes directly
* to output streams (e.g. std::cout), if ToString() can handle [adios2_type].
Expand Down
39 changes: 35 additions & 4 deletions source/adios2/core/ADIOS.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include "adios2/core/IO.h"
#include "adios2/helper/adiosCommDummy.h"
#include "adios2/helper/adiosFunctions.h" //InquireKey, BroadcastFile
#include "adios2/helper/adiosYAML.h"
#include "adios2/operator/OperatorFactory.h"
#include <adios2sys/SystemTools.hxx>

Expand Down Expand Up @@ -106,6 +107,10 @@ std::mutex PerfStubsMutex;
static std::atomic_uint adios_refcount(0); // adios objects at the same time
static std::atomic_uint adios_count(0); // total adios objects during runtime

/** User defined options from ~/.config/adios2/adios2.yaml if it exists */
static adios2::UserOptions UserOptions;
const adios2::UserOptions &ADIOS::GetUserOptions() { return UserOptions; };

ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string hostLanguage)
: m_HostLanguage(hostLanguage), m_Comm(std::move(comm)), m_ConfigFile(configFile),
m_CampaignManager(m_Comm)
Expand All @@ -124,6 +129,7 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string
}
}
#endif
ProcessUserConfig();
if (!configFile.empty())
{
if (!adios2sys::SystemTools::FileExists(configFile))
Expand All @@ -143,8 +149,11 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string
#ifdef ADIOS2_HAVE_KOKKOS
m_GlobalServices.Init_Kokkos_API();
#endif
std::string campaignName = "campaign_" + std::to_string(adios_count);
m_CampaignManager.Open(campaignName);
if (UserOptions.campaign.active)
{
std::string campaignName = "campaign_" + std::to_string(adios_count);
m_CampaignManager.Open(campaignName, UserOptions);
}
}

ADIOS::ADIOS(const std::string configFile, const std::string hostLanguage)
Expand All @@ -166,7 +175,26 @@ ADIOS::~ADIOS()
{
m_GlobalServices.Finalize();
}
m_CampaignManager.Close();
if (UserOptions.campaign.active)
{
m_CampaignManager.Close();
}
}

void ADIOS::ProcessUserConfig()
{
// read config parameters from config file
std::string homePath;
#ifdef _WIN32
homePath = getenv("HOMEPATH");
#else
homePath = getenv("HOME");
#endif
const std::string cfgFile = homePath + "/.config/adios2/adios2.yaml";
if (adios2sys::SystemTools::FileExists(cfgFile))
{
helper::ParseUserOptionsFile(m_Comm, cfgFile, UserOptions, homePath);
}
}

IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder)
Expand Down Expand Up @@ -330,7 +358,10 @@ void ADIOS::YAMLInitIO(const std::string &configFileYAML, const std::string &con

void ADIOS::RecordOutputStep(const std::string &name, const size_t step, const double time)
{
m_CampaignManager.Record(name, step, time);
if (UserOptions.campaign.active)
{
m_CampaignManager.Record(name, step, time);
}
}

void ADIOS::Global_init_AWS_API()
Expand Down
5 changes: 5 additions & 0 deletions source/adios2/core/ADIOS.h
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,9 @@ class ADIOS
void RecordOutputStep(const std::string &name, const size_t step = UnknownStep,
const double time = UnknownTime);

/** A constant reference to the user options from ~/.config/adios2/adios2.yaml */
static const adios2::UserOptions &GetUserOptions();

private:
/** Communicator given to parallel constructor. */
helper::Comm m_Comm;
Expand Down Expand Up @@ -204,6 +207,8 @@ class ADIOS
void YAMLInitIO(const std::string &configFileYAML, const std::string &configFileContents,
core::IO &io);

void ProcessUserConfig();

private:
/* Global services that we want to initialize at most once and shutdown
automatically when the ADIOS object is destructed. This only works
Expand Down
3 changes: 2 additions & 1 deletion source/adios2/core/Engine.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ namespace core

Engine::Engine(const std::string engineType, IO &io, const std::string &name, const Mode openMode,
helper::Comm comm)
: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm))
: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm)),
m_UserOptions(io.m_ADIOS.GetUserOptions())
{
m_FailVerbose = (m_Comm.Rank() == 0);
}
Expand Down
3 changes: 3 additions & 0 deletions source/adios2/core/Engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,9 @@ class Engine
* if no communicator is passed */
helper::Comm m_Comm;

/** User options parsed by the ADIOS object, here just for easy reference */
const UserOptions &m_UserOptions;

/** keeps track of current advance status */
StepStatus m_AdvanceStatus = StepStatus::OK;

Expand Down
33 changes: 16 additions & 17 deletions source/adios2/engine/campaign/CampaignData.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
#include <iostream>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <zlib.h>

namespace adios2
Expand Down Expand Up @@ -70,12 +69,13 @@ static int sqlcb_bpdataset(void *p, int argc, char **argv, char **azColName)
{
CampaignData *cdp = reinterpret_cast<CampaignData *>(p);
CampaignBPDataset cds;
size_t hostid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int");
size_t dirid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int");
size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int");
size_t hostid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int");
size_t dirid = helper::StringToSizeT(std::string(argv[2]), "SQL callback convert text to int");
cds.hostIdx = hostid - 1; // SQL rows start from 1, vector idx start from 0
cds.dirIdx = dirid - 1;
cds.name = argv[2];
cdp->bpdatasets.push_back(cds);
cds.name = argv[3];
cdp->bpdatasets[dsid] = cds;
return 0;
};

Expand All @@ -84,16 +84,15 @@ static int sqlcb_bpfile(void *p, int argc, char **argv, char **azColName)
CampaignData *cdp = reinterpret_cast<CampaignData *>(p);
CampaignBPFile cf;
size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int");
cf.bpDatasetIdx = dsid - 1;
cf.bpDatasetIdx = dsid;
cf.name = std::string(argv[1]);
int comp = helper::StringTo<int>(std::string(argv[2]), "SQL callback convert text to int");
cf.compressed = (bool)comp;
cf.lengthOriginal =
helper::StringToSizeT(std::string(argv[3]), "SQL callback convert text to int");
cf.lengthCompressed =
helper::StringToSizeT(std::string(argv[4]), "SQL callback convert text to int");
cf.ctime = static_cast<long>(
helper::StringTo<int64_t>(std::string(argv[5]), "SQL callback convert ctime to int"));
cf.ctime = helper::StringTo<int64_t>(std::string(argv[5]), "SQL callback convert ctime to int");

CampaignBPDataset &cds = cdp->bpdatasets[cf.bpDatasetIdx];
cds.files.push_back(cf);
Expand Down Expand Up @@ -139,7 +138,7 @@ void ReadCampaignData(sqlite3 *db, CampaignData &cd)
sqlite3_free(zErrMsg);
}

sqlcmd = "SELECT hostid, dirid, name FROM bpdataset";
sqlcmd = "SELECT rowid, hostid, dirid, name FROM bpdataset";
rc = sqlite3_exec(db, sqlcmd.c_str(), sqlcb_bpdataset, &cd, &zErrMsg);
if (rc != SQLITE_OK)
{
Expand Down Expand Up @@ -233,9 +232,9 @@ int inflateToFile(const unsigned char *source, const size_t blobsize, std::ofstr
return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
}

static long timeToSec(long ct)
static int64_t timeToSec(int64_t ct)
{
long t;
int64_t t;
if (ct > 99999999999999999)
{
/* nanosec to sec */
Expand All @@ -258,7 +257,7 @@ static long timeToSec(long ct)
return t;
}

static bool isFileNewer(const std::string path, long ctime)
static bool isFileNewer(const std::string path, int64_t ctime)
{
int result;
#ifdef _WIN32
Expand All @@ -273,9 +272,9 @@ static bool isFileNewer(const std::string path, long ctime)
return false;
}

long ct = s.st_ctime;
long ctSec = timeToSec(ct);
long ctimeSec = timeToSec(ctime);
int64_t ct = static_cast<int64_t>(s.st_ctime);
int64_t ctSec = timeToSec(ct);
int64_t ctimeSec = timeToSec(ctime);

/*std::cout << " Stat(" << path << "): size = " << s.st_size
<< " ct = " << ctSec << " ctime = " << ctimeSec << "\n";*/
Expand All @@ -292,13 +291,13 @@ void SaveToFile(sqlite3 *db, const std::string &path, const CampaignBPFile &bpfi
int rc;
char *zErrMsg = 0;
std::string sqlcmd;
std::string id = std::to_string(bpfile.bpDatasetIdx + 1);
std::string id = std::to_string(bpfile.bpDatasetIdx);

sqlite3_stmt *statement;
sqlcmd =
"SELECT data FROM bpfile WHERE bpdatasetid = " + id + " AND name = '" + bpfile.name + "'";
// std::cout << "SQL statement: " << sqlcmd << "\n";
rc = sqlite3_prepare_v2(db, sqlcmd.c_str(), sqlcmd.size(), &statement, NULL);
rc = sqlite3_prepare_v2(db, sqlcmd.c_str(), static_cast<int>(sqlcmd.size()), &statement, NULL);
if (rc != SQLITE_OK)
{
std::cout << "SQL error: " << zErrMsg << std::endl;
Expand Down
8 changes: 4 additions & 4 deletions source/adios2/engine/campaign/CampaignData.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@

#include <fstream>
#include <iostream>
#include <map>
#include <string>
#include <unordered_map>
#include <vector>

#include <sqlite3.h>
Expand All @@ -37,11 +37,11 @@ struct CampaignHost
struct CampaignBPFile
{
std::string name;
size_t bpDatasetIdx;
size_t bpDatasetIdx; // index of parent CampaignBPDataset in the map
bool compressed;
size_t lengthOriginal;
size_t lengthCompressed;
long ctime;
int64_t ctime;
};

struct CampaignBPDataset
Expand All @@ -56,7 +56,7 @@ struct CampaignData
{
std::string version;
std::vector<CampaignHost> hosts;
std::vector<CampaignBPDataset> bpdatasets;
std::map<size_t, CampaignBPDataset> bpdatasets;
};

void ReadCampaignData(sqlite3 *db, CampaignData &cd);
Expand Down
32 changes: 14 additions & 18 deletions source/adios2/engine/campaign/CampaignManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name
"SQL error on writing records:");
sqlite3_free(zErrMsg);
}
sqlcmd = "CREATE TABLE if not exists bpfiles (name);";
sqlcmd = "CREATE TABLE if not exists bpfiles (name PRIMARY KEY);";
rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg);
if (rc != SQLITE_OK)
{
Expand All @@ -56,7 +56,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name

for (auto &r : cmap)
{
sqlcmd = "INSERT INTO bpfiles (name)\n";
sqlcmd = "INSERT OR IGNORE INTO bpfiles (name)\n";
sqlcmd += "VALUES('" + r.first + "');";
rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg);
if (rc != SQLITE_OK)
Expand All @@ -74,31 +74,27 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name
return 0;
}

CampaignManager::CampaignManager(adios2::helper::Comm &comm)
{
m_WriterRank = comm.Rank();
if (m_Verbosity == 5)
{
std::cout << "Campaign Manager " << m_WriterRank << " constructor called" << std::endl;
}
}
CampaignManager::CampaignManager(adios2::helper::Comm &comm) { m_WriterRank = comm.Rank(); }

CampaignManager::~CampaignManager()
{
if (m_Verbosity == 5)
{
std::cout << "Campaign Manager " << m_WriterRank << " desctructor called\n";
}
if (m_Opened)
{
Close();
}
}

void CampaignManager::Open(const std::string &name)
void CampaignManager::Open(const std::string &name, const UserOptions &options)
{
m_Name = m_CampaignDir + "/" + name + "_" + std::to_string(m_WriterRank);
if (m_Verbosity == 5)
const UserOptions::Campaign &opts = options.campaign;
m_Options.active = opts.active;
m_Options.hostname = opts.hostname;
m_Options.campaignstorepath = opts.campaignstorepath;
m_Options.cachepath = opts.cachepath;
m_Options.verbose = opts.verbose;

m_Name = m_CampaignDir + PathSeparator + name + "_" + std::to_string(m_WriterRank);
if (m_Options.verbose > 0)
{
std::cout << "Campaign Manager " << m_WriterRank << " Open(" << m_Name << ")\n";
}
Expand All @@ -107,7 +103,7 @@ void CampaignManager::Open(const std::string &name)

void CampaignManager::Record(const std::string &name, const size_t step, const double time)
{
if (m_Verbosity == 5)
if (m_Options.verbose > 0)
{
std::cout << "Campaign Manager " << m_WriterRank << " Record name = " << name
<< " step = " << step << " time = " << time << "\n";
Expand Down
Loading

0 comments on commit 56320ec

Please sign in to comment.