Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

match type of timestep for h5 engine to size_t (same as adios Variabl… #3776

Merged
merged 5 commits into from
Aug 23, 2023
Merged

match type of timestep for h5 engine to size_t (same as adios Variabl… #3776

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
29bf70f
match type of timestep for h5 engine to size_t (same as adios Variabl…
guj Aug 22, 2023
702e298
clang-format
guj Aug 22, 2023
3907568
removing warning from auto
guj Aug 22, 2023
56f06b1
removed comments and cleaned up more code
guj Aug 23, 2023
3502c55
removed a comment
guj Aug 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 5 additions & 46 deletions source/adios2/engine/hdf5/HDF5ReaderP.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,14 +77,7 @@ void HDF5ReaderP::Init()
m_H5File.ParseParameters(m_IO);

/*
int ts = m_H5File.GetNumAdiosSteps();

if (ts == 0)
{
helper::Throw<std::runtime_error>( "Engine", "HDF5ReaderP", "Init",
"This h5 file is NOT written by ADIOS2");
}
*/
*/
m_H5File.ReadAttrToIO(m_IO);
if (!m_InStreamMode)
{
Expand All @@ -103,37 +96,6 @@ void HDF5ReaderP::Init()
template <class T>
size_t HDF5ReaderP::ReadDataset(hid_t dataSetId, hid_t h5Type, Variable<T> &variable, T *values)
{
/*
size_t variableStart = variable.m_StepsStart;

if ((m_H5File.m_IsGeneratedByAdios) && (ts >= 0))
{
try
{
m_H5File.SetAdiosStep(variableStart + ts);
}
catch (std::exception &e)
{
printf("[Not fatal] %s\n", e.what());
return 0;
//break;
}
}

std::vector<hid_t> chain;
if (!m_H5File.OpenDataset(variable.m_Name, chain)) {
return -1;
}

hid_t dataSetId = chain.back();
interop::HDF5DatasetGuard g(chain);
//hid_t dataSetId =
// H5Dopen(m_H5File.m_GroupId, variable.m_Name.c_str(), H5P_DEFAULT);
if (dataSetId < 0)
{
return 0;
}
*/
hid_t fileSpace = H5Dget_space(dataSetId);
interop::HDF5TypeGuard g_fs(fileSpace, interop::E_H5_SPACE);

Expand Down Expand Up @@ -248,8 +210,8 @@ void HDF5ReaderP::UseHDFRead(Variable<T> &variable, T *data, hid_t h5Type)
}

T *values = data;
unsigned int ts = 0;
// T *values = data;
// unsigned int ts = 0;
size_t ts = 0;
size_t variableStart = variable.m_StepsStart;
/*
// looks like m_StepsStart is defaulted to be 0 now.
Expand Down Expand Up @@ -278,8 +240,7 @@ void HDF5ReaderP::UseHDFRead(Variable<T> &variable, T *data, hid_t h5Type)
}
hid_t dataSetId = chain.back();
interop::HDF5DatasetGuard g(chain);
// hid_t dataSetId =
// H5Dopen(m_H5File.m_GroupId, variable.m_Name.c_str(), H5P_DEFAULT);

if (dataSetId < 0)
{
return;
Expand All @@ -291,8 +252,6 @@ void HDF5ReaderP::UseHDFRead(Variable<T> &variable, T *data, hid_t h5Type)
{
break;
}
// H5Sclose(fileSpace);
// H5Dclose(dataSetId);

ts++;
values += slabsize;
Expand All @@ -304,7 +263,7 @@ void HDF5ReaderP::UseHDFRead(Variable<T> &variable, T *data, hid_t h5Type)

StepStatus HDF5ReaderP::BeginStep(StepMode mode, const float timeoutSeconds)
{
const unsigned int ts = m_H5File.GetNumAdiosSteps();
const size_t ts = m_H5File.GetNumAdiosSteps();

if (m_StreamAt >= ts)
{
Expand Down
3 changes: 2 additions & 1 deletion source/adios2/engine/hdf5/HDF5ReaderP.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ class HDF5ReaderP : public Engine
void Init() final;

bool m_InStreamMode = false; // default is not streaming, i.e. set var timestep range
unsigned int m_StreamAt = 0; // stream step counter
// unsigned int m_StreamAt = 0; // stream step counter
size_t m_StreamAt = 0;
#define declare_type(T) \
void DoGetSync(Variable<T> &, T *) final; \
void DoGetDeferred(Variable<T> &, T *) final; \
Expand Down
50 changes: 31 additions & 19 deletions source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include <complex>
#include <ios>
#include <iostream>
#include <limits>
#include <mutex>
#include <stdexcept>
#include <vector>
Expand Down Expand Up @@ -85,6 +86,14 @@ HDF5Common::HDF5Common()
H5T_NATIVE_LDOUBLE);

m_PropertyTxfID = H5Pcreate(H5P_DATASET_XFER);

size_t size_t_max = (size_t)-1;
if (size_t_max <= std::numeric_limits<unsigned int>::max())
m_TimeStepH5T = H5T_NATIVE_UINT;
else if (size_t_max <= std::numeric_limits<unsigned long>::max())
m_TimeStepH5T = H5T_NATIVE_ULONG;

// default is m_TimeStepH5T = H5T_NATIVE_ULLONG;
}

HDF5Common::~HDF5Common() { Close(); }
Expand Down Expand Up @@ -279,32 +288,36 @@ void HDF5Common::WriteAdiosSteps()

hid_t s = H5Screate(H5S_SCALAR);
hid_t attr = H5Aexists(m_FileId, ATTRNAME_NUM_STEPS.c_str());

if (0 == attr)
attr = H5Acreate(m_FileId, ATTRNAME_NUM_STEPS.c_str(), H5T_NATIVE_UINT, s, H5P_DEFAULT,
attr = H5Acreate(m_FileId, ATTRNAME_NUM_STEPS.c_str(), m_TimeStepH5T, s, H5P_DEFAULT,
H5P_DEFAULT);
else
attr = H5Aopen(m_FileId, ATTRNAME_NUM_STEPS.c_str(), H5P_DEFAULT);

unsigned int totalAdiosSteps = m_CurrentAdiosStep + 1;
// unsigned int totalAdiosSteps = m_CurrentAdiosStep + 1;
size_t totalAdiosSteps = m_CurrentAdiosStep + 1;

if (m_GroupId < 0)
{
totalAdiosSteps = m_CurrentAdiosStep;
}

H5Awrite(attr, H5T_NATIVE_UINT, &totalAdiosSteps);
H5Awrite(attr, m_TimeStepH5T, &totalAdiosSteps);

H5Sclose(s);
H5Aclose(attr);
}

unsigned int HDF5Common::GetAdiosStep() const { return m_NumAdiosSteps; }
size_t HDF5Common::GetAdiosStep() const { return m_NumAdiosSteps; }

unsigned int HDF5Common::GetNumAdiosSteps()
// A valid file should have at least 1 step.
size_t HDF5Common::GetNumAdiosSteps()
{
if (m_WriteMode)
{
return static_cast<unsigned int>(-1);
// return static_cast<unsigned int>(-1);
return 0;
}

if (m_FileId < 0)
Expand All @@ -329,7 +342,7 @@ unsigned int HDF5Common::GetNumAdiosSteps()
{
hid_t attr = H5Aopen(m_FileId, ATTRNAME_NUM_STEPS.c_str(), H5P_DEFAULT);

H5Aread(attr, H5T_NATIVE_UINT, &m_NumAdiosSteps);
H5Aread(attr, m_TimeStepH5T, &m_NumAdiosSteps);
H5Aclose(attr);
}
}
Expand All @@ -347,9 +360,9 @@ void HDF5Common::ReadAllVariables(core::IO &io)
}

GetNumAdiosSteps();
unsigned int i = 0;
// unsigned int i = 0;

for (i = 0; i < m_NumAdiosSteps; i++)
for (size_t i = 0; i < m_NumAdiosSteps; i++)
{
ReadVariables(i, io);
}
Expand All @@ -362,7 +375,6 @@ void HDF5Common::FindVarsFromH5(core::IO &io, hid_t top_id, const char *gname, c
// std::string stepStr;
hsize_t numObj;

// StaticGetAdiosStepString(stepStr, ts);
hid_t gid = H5Gopen2(top_id, gname, H5P_DEFAULT);
HDF5TypeGuard g(gid, E_H5_GROUP);
/// if (gid > 0) {
Expand Down Expand Up @@ -798,7 +810,7 @@ void HDF5Common::Close()
m_GroupId = -1;
}

void HDF5Common::SetAdiosStep(int step)
void HDF5Common::SetAdiosStep(size_t step)
{
if (m_WriteMode)
helper::Throw<std::ios_base::failure>("Toolkit", "interop::hdf5::HDF5Common",
Expand All @@ -811,13 +823,13 @@ void HDF5Common::SetAdiosStep(int step)

GetNumAdiosSteps();

unsigned int ustep = static_cast<unsigned int>(step);
if (ustep >= m_NumAdiosSteps)
// unsigned int ustep = static_cast<unsigned int>(step);
if (step >= m_NumAdiosSteps)
helper::Throw<std::ios_base::failure>("Toolkit", "interop::hdf5::HDF5Common",
"SetAdiosStep",
"given time step is more than actual known steps");

if (m_CurrentAdiosStep == ustep)
if (m_CurrentAdiosStep == step)
{
return;
}
Expand All @@ -835,7 +847,7 @@ void HDF5Common::SetAdiosStep(int step)
"ERROR: unable to open HDF5 group " + stepName + ", in call to Open");
}

m_CurrentAdiosStep = ustep;
m_CurrentAdiosStep = step;
}

//
Expand Down Expand Up @@ -896,11 +908,11 @@ void HDF5Common::Advance()
}
else
{
if (m_NumAdiosSteps == 0)
if (0 == m_NumAdiosSteps)
{
GetNumAdiosSteps();
}
if (m_CurrentAdiosStep + 1 >= m_NumAdiosSteps)
if ((m_CurrentAdiosStep + 1) >= m_NumAdiosSteps)
{
return;
}
Expand Down Expand Up @@ -1467,7 +1479,7 @@ void HDF5Common::LocateAttrParent(const std::string &attrName, std::vector<std::
if (list.size() >= 1)
{
std::string ts;
for (unsigned int i = 0; i < m_CurrentAdiosStep; i++)
for (size_t i = 0; i < m_CurrentAdiosStep; i++)
{
StaticGetAdiosStepString(ts, i);
for (size_t j = 0; j < list.size() - 1; j++)
Expand Down Expand Up @@ -1725,7 +1737,7 @@ void HDF5Common::ReadNativeAttrToIO(core::IO &io, hid_t datasetId, std::string c
}
}

void HDF5Common::StaticGetAdiosStepString(std::string &stepName, int ts)
void HDF5Common::StaticGetAdiosStepString(std::string &stepName, size_t ts)
{
stepName = "/Step" + std::to_string(ts);
}
Expand Down
17 changes: 11 additions & 6 deletions source/adios2/toolkit/interop/hdf5/HDF5Common.h
View file
Open in desktop
eisenhauer marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -172,10 +172,11 @@ class HDF5Common

void ReadNativeAttrToIO(core::IO &io, hid_t datasetId, std::string const &pathFromRoot);

void SetAdiosStep(int ts);
// void SetAdiosStep(int ts);
void SetAdiosStep(size_t ts);

unsigned int GetNumAdiosSteps();
unsigned int GetAdiosStep() const;
size_t GetNumAdiosSteps();
size_t GetAdiosStep() const;
void WriteAdiosSteps();

void ReadVariables(unsigned int ts, core::IO &io);
Expand All @@ -200,7 +201,7 @@ class HDF5Common
void AddNonStringAttribute(core::IO &io, std::string const &attrName, hid_t attrId,
hid_t h5Type, hsize_t arraySize);

static void StaticGetAdiosStepString(std::string &adiosStepName, int ts);
static void StaticGetAdiosStepString(std::string &adiosStepName, size_t ts);

hid_t m_PropertyListId = -1;
hid_t m_PropertyTxfID = -1;
Expand All @@ -212,7 +213,8 @@ class HDF5Common
hid_t m_DefH5TypeComplexFloat;
hid_t m_DefH5TypeBlockStat;

unsigned int m_CurrentAdiosStep = 0;
// unsigned int m_CurrentAdiosStep = 0;
size_t m_CurrentAdiosStep = 0;

void CheckWriteGroup();

Expand Down Expand Up @@ -247,7 +249,8 @@ class HDF5Common
std::vector<hsize_t> &, std::vector<hsize_t> &);

bool m_WriteMode = false;
unsigned int m_NumAdiosSteps = 0;
// unsigned int m_NumAdiosSteps = 0;
size_t m_NumAdiosSteps = 0;

MPI_API const *m_MPI = nullptr;
int m_CommRank = 0;
Expand All @@ -258,6 +261,8 @@ class HDF5Common
template <class T>
void AddStats(const core::Variable<T> &variable, hid_t parentId, std::vector<T> &stats);

hid_t m_TimeStepH5T = H5T_NATIVE_ULLONG;
;
hid_t m_ChunkPID;
int m_ChunkDim;
std::set<std::string> m_ChunkVarNames;
Expand Down