From 30c6dd112ad38c667755fb7aa23a7420a3441247 Mon Sep 17 00:00:00 2001 From: williamfgc Date: Wed, 30 May 2018 10:32:46 -0400 Subject: [PATCH] #656 Reverting HDF5 tests --- .../adios2/engine/hdf5/TestHDF5WriteRead.cpp | 2402 +++++++++++++++-- 1 file changed, 2138 insertions(+), 264 deletions(-) diff --git a/testing/adios2/engine/hdf5/TestHDF5WriteRead.cpp b/testing/adios2/engine/hdf5/TestHDF5WriteRead.cpp index 7fe57c8908..12b85c9df3 100644 --- a/testing/adios2/engine/hdf5/TestHDF5WriteRead.cpp +++ b/testing/adios2/engine/hdf5/TestHDF5WriteRead.cpp @@ -37,6 +37,7 @@ class HDF5NativeReader hid_t &h5Type); // If offset, count and memspaceSize are provided, then variable would be // read by selection + void ReadString(const std::string varName, std::string &result); void ReadVar(const std::string varName, void *dataArray, hsize_t *offset = nullptr, hsize_t *count = nullptr, const size_t memsspaceSize = 0); @@ -50,6 +51,231 @@ class HDF5NativeReader hid_t m_GroupId; }; +class HDF5NativeWriter +{ +public: + HDF5NativeWriter(const std::string fileName, MPI_Comm comm); + ~HDF5NativeWriter(); + + void Advance(); + + void CreateAndStoreScalar(std::string const &variableName, hid_t h5Type, + const void *values); + void CreateAndStoreVar(std::string const &variableName, int dimSize, + hid_t h5Type, const hsize_t *global_dims, + const hsize_t *offsets, const hsize_t *counts, + const void *values); + + /* + void WriteVar(const std::string varName, void *dataArray, + hsize_t *offset = nullptr, hsize_t *count = nullptr, + const size_t memsspaceSize = 0); + */ + int m_CurrentTimeStep; + unsigned int m_TotalTimeSteps; + +private: + void CheckWriteGroup(); + + hid_t m_FilePropertyListId; + hid_t m_FileId; + hid_t m_GroupId; +}; + +HDF5NativeWriter::HDF5NativeWriter(const std::string fileName, MPI_Comm comm) +: m_CurrentTimeStep(0), m_TotalTimeSteps(0) +{ + m_FilePropertyListId = H5Pcreate(H5P_FILE_ACCESS); + +#ifdef ADIOS2_HAVE_MPI + H5Pset_fapl_mpio(m_FilePropertyListId, comm, MPI_INFO_NULL); +#endif + + // std::string ts0 = "/AdiosStep0"; + // stepName = "/Step" + std::to_string(ts); + std::string ts0 = "/Step0"; + + /* + * Create a new file collectively and release property list identifier. + */ + m_FileId = H5Fcreate(fileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, + m_FilePropertyListId); + if (m_FileId < 0) + { + throw std::runtime_error("Unable to create file: " + fileName); + } + + m_GroupId = H5Gcreate2(m_FileId, ts0.c_str(), H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + + if (m_GroupId < 0) + { + throw std::runtime_error("ERROR: Unable to create HDF5 group " + ts0); + } +} + +HDF5NativeWriter::~HDF5NativeWriter() +{ + if (m_FileId < 0) + { + return; + } + + // write NumStep attr + hid_t s = H5Screate(H5S_SCALAR); + + hid_t attr = H5Acreate(m_FileId, "NumSteps", H5T_NATIVE_UINT, s, + H5P_DEFAULT, H5P_DEFAULT); + uint totalAdiosSteps = m_CurrentTimeStep + 1; + + if (m_GroupId < 0) + { + totalAdiosSteps = m_CurrentTimeStep; + } + + H5Awrite(attr, H5T_NATIVE_UINT, &totalAdiosSteps); + + H5Sclose(s); + H5Aclose(attr); + + // now close necessary ids + if (m_GroupId >= 0) + { + H5Gclose(m_GroupId); + } + + H5Fclose(m_FileId); + H5Pclose(m_FilePropertyListId); +} + +void HDF5NativeWriter::CheckWriteGroup() +{ + if (m_GroupId >= 0) + { + return; + } + + std::string stepName = "/Step" + std::to_string(m_CurrentTimeStep); + + m_GroupId = H5Gcreate2(m_FileId, stepName.c_str(), H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + + if (m_GroupId < 0) + { + throw std::runtime_error("ERROR: Unable to create HDF5 group " + + stepName); + } +} + +void HDF5NativeWriter::CreateAndStoreScalar(std::string const &variableName, + hid_t h5Type, const void *values) +{ + CheckWriteGroup(); + + // write scalar + hid_t filespaceID = H5Screate(H5S_SCALAR); + hid_t plistID = H5Pcreate(H5P_DATASET_XFER); +#ifdef ADIOS2_HAVE_MPI + H5Pset_dxpl_mpio(plistID, H5FD_MPIO_COLLECTIVE); +#endif + + hid_t dsetID; + + if (h5Type != H5T_STRING) + { + dsetID = H5Dcreate(m_GroupId, variableName.c_str(), h5Type, filespaceID, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + herr_t status = + H5Dwrite(dsetID, h5Type, H5S_ALL, H5S_ALL, plistID, values); + } + else + { + /* Create a datatype to refer to. */ + hid_t type = H5Tcopy(H5T_C_S1); + char *strval = (char *)values; + hid_t ret = H5Tset_size(type, strlen(strval)); + + ret = H5Tset_strpad(type, H5T_STR_NULLTERM); + + /* Test creating a "normal" sized string attribute */ + dsetID = H5Dcreate(m_GroupId, variableName.c_str(), type, filespaceID, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + ret = H5Dwrite(dsetID, type, H5S_ALL, H5S_ALL, plistID, values); + +#ifdef DOUBLECHECK + size_t typesize = H5Tget_size(type); + char *val = (char *)(calloc(typesize, sizeof(char))); + + hid_t ret2 = H5Dread(dsetID, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, val); + std::cerr << " .... typesize=" << typesize << " val=" << val + << std::endl; + free val; +#endif + } + + H5Sclose(filespaceID); + H5Dclose(dsetID); +} + +void HDF5NativeWriter::CreateAndStoreVar(std::string const &variableName, + int dimSize, hid_t h5Type, + const hsize_t *global_dims, + const hsize_t *offsets, + const hsize_t *counts, + const void *values) +{ + if (h5Type == H5T_STRING) + { + throw std::runtime_error("Sync with ADIOS2. It does not store string " + "var with dimensions yet!"); + } + + CheckWriteGroup(); + hid_t fileSpace = H5Screate_simple(dimSize, global_dims, NULL); + + hid_t dsetID = H5Dcreate(m_GroupId, variableName.c_str(), h5Type, fileSpace, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + hid_t memSpace = H5Screate_simple(dimSize, counts, NULL); + + // Select hyperslab + fileSpace = H5Dget_space(dsetID); + H5Sselect_hyperslab(fileSpace, H5S_SELECT_SET, offsets, NULL, counts, NULL); + + // Create property list for collective dataset write. + + hid_t plistID = H5Pcreate(H5P_DATASET_XFER); +#ifdef ADIOS2_HAVE_MPI + H5Pset_dxpl_mpio(plistID, H5FD_MPIO_COLLECTIVE); +#endif + herr_t status = + H5Dwrite(dsetID, h5Type, memSpace, fileSpace, plistID, values); + + if (status < 0) + { + throw std::runtime_error( + "ERROR: HDF5 file Write failed, in call to Write\n"); + } + + H5Dclose(dsetID); + H5Sclose(fileSpace); + H5Sclose(memSpace); + H5Pclose(plistID); +} + +void HDF5NativeWriter::Advance() +{ + if (m_GroupId >= 0) + { + H5Gclose(m_GroupId); + m_GroupId = -1; + } + ++m_CurrentTimeStep; +} + +// +// +// HDF5NativeReader::HDF5NativeReader(const std::string fileName) : m_CurrentTimeStep(0), m_TotalTimeSteps(0) { @@ -156,6 +382,34 @@ bool HDF5NativeReader::Advance() return true; } +void HDF5NativeReader::ReadString(const std::string varName, + std::string &result) +{ + if (m_GroupId < 0) + { + throw std::runtime_error("Can't read variable " + varName + + " since a group is not currently open"); + } + + hid_t dataSetId = H5Dopen(m_GroupId, varName.c_str(), H5P_DEFAULT); + if (dataSetId < 0) + { + throw std::runtime_error("Unable to open dataset " + varName + + "when ReadVar"); + } + + hid_t h5Type = H5Dget_type(dataSetId); + size_t typesize = H5Tget_size(h5Type); + + char *val = (char *)(calloc(typesize, sizeof(char))); + hid_t ret2 = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, val); + + result.assign(val); + free(val); + + H5Dclose(dataSetId); +} + void HDF5NativeReader::ReadVar(const std::string varName, void *dataArray, hsize_t *offset, hsize_t *count, const size_t memspaceSize) @@ -172,7 +426,6 @@ void HDF5NativeReader::ReadVar(const std::string varName, void *dataArray, throw std::runtime_error("Unable to open dataset " + varName + "when ReadVar"); } - hid_t fileSpace = H5Dget_space(dataSetId); if (fileSpace < 0) { @@ -254,6 +507,8 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read1D8) adios2::Dims shape{static_cast(Nx * mpiSize)}; adios2::Dims start{static_cast(Nx * mpiRank)}; adios2::Dims count{static_cast(Nx)}; + + io.DefineVariable("iString"); io.DefineVariable("i8", shape, start, count); io.DefineVariable("i16", shape, start, count); io.DefineVariable("i32", shape, start, count); @@ -285,6 +540,7 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read1D8) generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); // Retrieve the variables that previously went out of scope + auto &var_iString = *io.InquireVariable("iString"); auto &var_i8 = *io.InquireVariable("i8"); auto &var_i16 = *io.InquireVariable("i16"); auto &var_i32 = *io.InquireVariable("i32"); @@ -313,7 +569,8 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read1D8) // Write each one // fill in the variable with values from starting index to // starting index + count - // engine.BeginStep(); + engine.BeginStep(); + engine.Put(var_iString, currentTestData.S1); engine.Put(var_i8, currentTestData.I8.data()); engine.Put(var_i16, currentTestData.I16.data()); engine.Put(var_i32, currentTestData.I32.data()); @@ -335,6 +592,7 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read1D8) if (doRead) { const size_t arraySize = Nx; + std::string IString; std::array I8; std::array I16; std::array I32; @@ -349,8 +607,8 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read1D8) HDF5NativeReader hdf5Reader(fname); // 1D hsize_t count[1], offset[1]; - count[0] = mpiRank * Nx; - offset[0] = Nx; + offset[0] = mpiRank * Nx; + count[0] = Nx; size_t globalArraySize = Nx * mpiSize; // For each variable, we would verify its global size and type. @@ -364,65 +622,73 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read1D8) std::vector gDims; hid_t h5Type; + // auto var_iString = io.InquireVariable("iString"); + hdf5Reader.GetVarInfo("iString", gDims, h5Type); + // ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_IN), 1); + ASSERT_EQ(gDims.size(), 0); + hdf5Reader.ReadString("iString", IString); + hdf5Reader.GetVarInfo("i8", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_INT8), 1); ASSERT_EQ(gDims.size(), 1); ASSERT_EQ(gDims[0], globalArraySize); - hdf5Reader.ReadVar("i8", I8.data(), count, offset, arraySize); + hdf5Reader.ReadVar("i8", I8.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("i16", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_SHORT), 1); ASSERT_EQ(gDims.size(), 1); ASSERT_EQ(gDims[0], globalArraySize); - hdf5Reader.ReadVar("i16", I16.data(), count, offset, arraySize); + hdf5Reader.ReadVar("i16", I16.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("i32", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_INT), 1); ASSERT_EQ(gDims.size(), 1); ASSERT_EQ(gDims[0], globalArraySize); - hdf5Reader.ReadVar("i32", I32.data(), count, offset, arraySize); + hdf5Reader.ReadVar("i32", I32.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("i64", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_LONG), 1); ASSERT_EQ(gDims.size(), 1); ASSERT_EQ(gDims[0], globalArraySize); - hdf5Reader.ReadVar("i64", I64.data(), count, offset, arraySize); + hdf5Reader.ReadVar("i64", I64.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("u8", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_UCHAR), 1); ASSERT_EQ(gDims.size(), 1); ASSERT_EQ(gDims[0], globalArraySize); - hdf5Reader.ReadVar("u8", U8.data(), count, offset, arraySize); + hdf5Reader.ReadVar("u8", U8.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("u16", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_USHORT), 1); ASSERT_EQ(gDims.size(), 1); ASSERT_EQ(gDims[0], globalArraySize); - hdf5Reader.ReadVar("u16", U16.data(), count, offset, arraySize); + hdf5Reader.ReadVar("u16", U16.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("u32", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_UINT), 1); ASSERT_EQ(gDims.size(), 1); ASSERT_EQ(gDims[0], globalArraySize); - hdf5Reader.ReadVar("u32", U32.data(), count, offset, arraySize); + hdf5Reader.ReadVar("u32", U32.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("u64", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_ULONG), 1); ASSERT_EQ(gDims.size(), 1); ASSERT_EQ(gDims[0], globalArraySize); - hdf5Reader.ReadVar("u64", U64.data(), count, offset, arraySize); + hdf5Reader.ReadVar("u64", U64.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("r32", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_FLOAT), 1); ASSERT_EQ(gDims.size(), 1); ASSERT_EQ(gDims[0], globalArraySize); - hdf5Reader.ReadVar("r32", R32.data(), count, offset, arraySize); + hdf5Reader.ReadVar("r32", R32.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("r64", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_DOUBLE), 1); ASSERT_EQ(gDims.size(), 1); ASSERT_EQ(gDims[0], globalArraySize); - hdf5Reader.ReadVar("r64", R64.data(), count, offset, arraySize); + hdf5Reader.ReadVar("r64", R64.data(), offset, count, arraySize); + + EXPECT_EQ(IString, currentTestData.S1); // Check if it's correct for (size_t i = 0; i < Nx; ++i) @@ -448,38 +714,15 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read1D8) } // ADIOS2 write, ADIOS2 read -TEST_F(HDF5WriteReadTest, DISABLED_ADIOS2HDF5WriteADIOS2HDF5Read1D8) -{ - // std::string fname = "ADIOS2HDF5WriteADIOS2HDF5Read1D8.h5"; - - ASSERT_TRUE(false) << "ADIOS2 read API is not yet implemented"; -} - -// Native HDF5 write, ADIOS2 read -TEST_F(HDF5WriteReadTest, DISABLED_HDF5WriteADIOS2HDF5Read1D8) -{ - // std::string fname = "HDF5WriteADIOS2HDF5Read1D8.h5"; - - ASSERT_TRUE(false) << "ADIOS2 read API is not yet implemented"; -} - -//****************************************************************************** -// 2D 2x4 test data -//****************************************************************************** - -// ADIOS2 write, native HDF5 read -TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D2x4) +TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteADIOS2HDF5Read1D8) { - // Each process would write a 2x4 array and all processes would - // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - std::string fname = "ADIOS2HDF5WriteHDF5Read2D2x4Test.h5"; + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + const std::string fname = "ADIOS2HDF5WriteADIOS2HDF5Read1D8.h5"; int mpiRank = 0, mpiSize = 1; // Number of rows - const std::size_t Nx = 4; - - // Number of rows - const std::size_t Ny = 2; + const std::size_t Nx = 8; // Number of steps const std::size_t NSteps = 3; @@ -489,78 +732,1441 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D2x4) MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); #endif - // Write test data using ADIOS2 - { +// Write test data using ADIOS2 + #ifdef ADIOS2_HAVE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); #else - adios2::ADIOS adios(true); + adios2::ADIOS adios(true); #endif - adios2::IO &io = adios.DeclareIO("TestIO"); - - // Declare 2D variables (Ny * (NumOfProcesses * Nx)) - // The local process' part (start, count) can be defined now or later - // before Write(). - { - adios2::Dims shape{static_cast(Ny), - static_cast(Nx * mpiSize)}; - adios2::Dims start{static_cast(0), - static_cast(mpiRank * Nx)}; - adios2::Dims count{static_cast(Ny), - static_cast(Nx)}; - auto &var_i8 = io.DefineVariable("i8", shape, start, count); - auto &var_i16 = - io.DefineVariable("i16", shape, start, count); - auto &var_i32 = - io.DefineVariable("i32", shape, start, count); - auto &var_i64 = - io.DefineVariable("i64", shape, start, count); - auto &var_u8 = - io.DefineVariable("u8", shape, start, count); - auto &var_u16 = - io.DefineVariable("u16", shape, start, count); - auto &var_u32 = - io.DefineVariable("u32", shape, start, count); - auto &var_u64 = - io.DefineVariable("u64", shape, start, count); - auto &var_r32 = - io.DefineVariable("r32", shape, start, count); - auto &var_r64 = - io.DefineVariable("r64", shape, start, count); - } + adios2::IO &io = adios.DeclareIO("TestIO"); - // Create the HDF5 Engine - io.SetEngine("HDF5"); + // Declare 1D variables (NumOfProcesses * Nx) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + adios2::Dims shape{static_cast(Nx * mpiSize)}; + adios2::Dims start{static_cast(Nx * mpiRank)}; + adios2::Dims count{static_cast(Nx)}; - // HDf5 engine calls the HDF5 common object that calls the hDF5 library. - // The IO functionality, SetParameters and AddTransports will be added - // in the future. For now `io.AddTransport("file", { - // "library", "MPI"}});` is omitted. - // }) - io.AddTransport("file"); + io.DefineVariable("iString"); + io.DefineVariable("i8", shape, start, count); + io.DefineVariable("i16", shape, start, count); + io.DefineVariable("i32", shape, start, count); + io.DefineVariable("i64", shape, start, count); + io.DefineVariable("u8", shape, start, count); + io.DefineVariable("u16", shape, start, count); + io.DefineVariable("u32", shape, start, count); + io.DefineVariable("u64", shape, start, count); + io.DefineVariable("r32", shape, start, count); + io.DefineVariable("r64", shape, start, count); + } - adios2::Engine &engine = io.Open(fname, adios2::Mode::Write); + // Create the HDF5 Engine + io.SetEngine("HDF5"); - for (size_t step = 0; step < NSteps; ++step) - { - // Generate test data for each process uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + // HDf5 engine calls the HDF5 common object that calls the hDF5 library. + // The IO functionality, SetParameters and AddTransports will be added + // in the future. For now `io.AddTransport("file", { + // "library", "MPI"}});` is omitted. + // }) + // io.AddTransport("File"); - // Retrieve the variables that previously went out of scope - auto &var_i8 = *io.InquireVariable("i8"); - auto &var_i16 = *io.InquireVariable("i16"); - auto &var_i32 = *io.InquireVariable("i32"); - auto &var_i64 = *io.InquireVariable("i64"); - auto &var_u8 = *io.InquireVariable("u8"); - auto &var_u16 = *io.InquireVariable("u16"); - auto &var_u32 = *io.InquireVariable("u32"); - auto &var_u64 = *io.InquireVariable("u64"); - auto &var_r32 = *io.InquireVariable("r32"); - auto &var_r64 = *io.InquireVariable("r64"); + adios2::Engine &engine = io.Open(fname, adios2::Mode::Write); - // Make a 2D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + auto &var_iString = *io.InquireVariable("iString"); + auto &var_i8 = *io.InquireVariable("i8"); + auto &var_i16 = *io.InquireVariable("i16"); + auto &var_i32 = *io.InquireVariable("i32"); + auto &var_i64 = *io.InquireVariable("i64"); + auto &var_u8 = *io.InquireVariable("u8"); + auto &var_u16 = *io.InquireVariable("u16"); + auto &var_u32 = *io.InquireVariable("u32"); + auto &var_u64 = *io.InquireVariable("u64"); + auto &var_r32 = *io.InquireVariable("r32"); + auto &var_r64 = *io.InquireVariable("r64"); + + // Make a 1D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + + adios2::Box sel({mpiRank * Nx}, {Nx}); + + EXPECT_THROW(var_iString.SetSelection(sel), std::invalid_argument); + + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + engine.BeginStep(); + engine.Put(var_iString, currentTestData.S1); + engine.Put(var_i8, currentTestData.I8.data()); + engine.Put(var_i16, currentTestData.I16.data()); + engine.Put(var_i32, currentTestData.I32.data()); + engine.Put(var_i64, currentTestData.I64.data()); + engine.Put(var_u8, currentTestData.U8.data()); + engine.Put(var_u16, currentTestData.U16.data()); + engine.Put(var_u32, currentTestData.U32.data()); + engine.Put(var_u64, currentTestData.U64.data()); + engine.Put(var_r32, currentTestData.R32.data()); + engine.Put(var_r64, currentTestData.R64.data()); + // Advance to the next time step + engine.EndStep(); + } + + // Close the file + engine.Close(); + + { + adios2::IO &io = adios.DeclareIO("HDF5ReadIO"); + io.SetEngine("HDF5"); + + adios2::Engine &hdf5Reader = io.Open(fname, adios2::Mode::Read); + + auto var_iString = io.InquireVariable("iString"); + ASSERT_NE(var_iString, nullptr); + ASSERT_EQ(var_iString->m_Shape.size(), 0); + ASSERT_EQ(var_iString->m_AvailableStepsCount, NSteps); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], mpiSize * Nx); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], mpiSize * Nx); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], mpiSize * Nx); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], mpiSize * Nx); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], mpiSize * Nx); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], mpiSize * Nx); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], mpiSize * Nx); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], mpiSize * Nx); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], mpiSize * Nx); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], mpiSize * Nx); + + // TODO: other types + + SmallTestData testData; + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{mpiRank * Nx}; + const adios2::Dims count{Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + for (size_t t = 0; t < NSteps; ++t) + { + var_i8->SetStepSelection({t, 1}); + var_i16->SetStepSelection({t, 1}); + var_i32->SetStepSelection({t, 1}); + var_i64->SetStepSelection({t, 1}); + + var_u8->SetStepSelection({t, 1}); + var_u16->SetStepSelection({t, 1}); + var_u32->SetStepSelection({t, 1}); + var_u64->SetStepSelection({t, 1}); + + var_r32->SetStepSelection({t, 1}); + var_r64->SetStepSelection({t, 1}); + + // Generate test data for each rank uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(t), mpiRank, mpiSize); + + hdf5Reader.BeginStep(); + hdf5Reader.Get(*var_iString, IString); + + hdf5Reader.Get(*var_i8, I8.data()); + hdf5Reader.Get(*var_i16, I16.data()); + hdf5Reader.Get(*var_i32, I32.data()); + hdf5Reader.Get(*var_i64, I64.data()); + + hdf5Reader.Get(*var_u8, U8.data()); + hdf5Reader.Get(*var_u16, U16.data()); + hdf5Reader.Get(*var_u32, U32.data()); + hdf5Reader.Get(*var_u64, U64.data()); + + hdf5Reader.Get(*var_r32, R32.data()); + hdf5Reader.Get(*var_r64, R64.data()); + + hdf5Reader.EndStep(); + + EXPECT_EQ(IString, currentTestData.S1); + + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + hdf5Reader.Close(); + } +} + +// Native HDF5 write, ADIOS2 read +TEST_F(HDF5WriteReadTest, HDF5WriteADIOS2HDF5Read1D8) +{ + std::string fname = "HDF5WriteADIOS2HDF5Read1D8.h5"; + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 8; + + // Number of steps + const std::size_t NSteps = 3; + + { +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + + HDF5NativeWriter h5writer(fname, MPI_COMM_WORLD); +#else + HDF5NativeWriter h5writer(fname, 0); +#endif + + int dimSize = 1; + hsize_t global_dims[1] = {Nx * mpiSize}; + hsize_t count[1] = {Nx}; + hsize_t offset[1] = {Nx * mpiRank}; + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + + h5writer.CreateAndStoreScalar("iString", H5T_STRING, + currentTestData.S1.data()); + h5writer.CreateAndStoreVar("i8", dimSize, H5T_NATIVE_INT8, + global_dims, offset, count, + currentTestData.I8.data()); + h5writer.CreateAndStoreVar("i16", dimSize, H5T_NATIVE_SHORT, + global_dims, offset, count, + currentTestData.I16.data()); + h5writer.CreateAndStoreVar("i32", dimSize, H5T_NATIVE_INT, + global_dims, offset, count, + currentTestData.I32.data()); + h5writer.CreateAndStoreVar("i64", dimSize, H5T_NATIVE_LONG, + global_dims, offset, count, + currentTestData.I64.data()); + h5writer.CreateAndStoreVar("u8", dimSize, H5T_NATIVE_UCHAR, + global_dims, offset, count, + currentTestData.U8.data()); + h5writer.CreateAndStoreVar("u16", dimSize, H5T_NATIVE_USHORT, + global_dims, offset, count, + currentTestData.U16.data()); + h5writer.CreateAndStoreVar("u32", dimSize, H5T_NATIVE_UINT, + global_dims, offset, count, + currentTestData.U32.data()); + h5writer.CreateAndStoreVar("u64", dimSize, H5T_NATIVE_ULONG, + global_dims, offset, count, + currentTestData.U64.data()); + h5writer.CreateAndStoreVar("r32", dimSize, H5T_NATIVE_FLOAT, + global_dims, offset, count, + currentTestData.R32.data()); + h5writer.CreateAndStoreVar("r64", dimSize, H5T_NATIVE_DOUBLE, + global_dims, offset, count, + currentTestData.R64.data()); + h5writer.Advance(); + } + } + + { // ADIOS2 read back + +// Write test data using ADIOS2 + +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + adios2::IO &io = adios.DeclareIO("HDF5ReadIO"); + io.SetEngine("HDF5"); + + adios2::Engine &hdf5Reader = io.Open(fname, adios2::Mode::Read); + + auto var_iString = io.InquireVariable("iString"); + ASSERT_NE(var_iString, nullptr); + ASSERT_EQ(var_iString->m_Shape.size(), 0); + ASSERT_EQ(var_iString->m_AvailableStepsCount, NSteps); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], mpiSize * Nx); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], mpiSize * Nx); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], mpiSize * Nx); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], mpiSize * Nx); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], mpiSize * Nx); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], mpiSize * Nx); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], mpiSize * Nx); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], mpiSize * Nx); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], mpiSize * Nx); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], mpiSize * Nx); + + // TODO: other types + + SmallTestData testData; + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{mpiRank * Nx}; + const adios2::Dims count{Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + for (size_t t = 0; t < NSteps; ++t) + { + var_i8->SetStepSelection({t, 1}); + var_i16->SetStepSelection({t, 1}); + var_i32->SetStepSelection({t, 1}); + var_i64->SetStepSelection({t, 1}); + + var_u8->SetStepSelection({t, 1}); + var_u16->SetStepSelection({t, 1}); + var_u32->SetStepSelection({t, 1}); + var_u64->SetStepSelection({t, 1}); + + var_r32->SetStepSelection({t, 1}); + var_r64->SetStepSelection({t, 1}); + + // Generate test data for each rank uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(t), mpiRank, mpiSize); + + hdf5Reader.BeginStep(); + hdf5Reader.Get(*var_iString, IString); + + hdf5Reader.Get(*var_i8, I8.data()); + hdf5Reader.Get(*var_i16, I16.data()); + hdf5Reader.Get(*var_i32, I32.data()); + hdf5Reader.Get(*var_i64, I64.data()); + + hdf5Reader.Get(*var_u8, U8.data()); + hdf5Reader.Get(*var_u16, U16.data()); + hdf5Reader.Get(*var_u32, U32.data()); + hdf5Reader.Get(*var_u64, U64.data()); + + hdf5Reader.Get(*var_r32, R32.data()); + hdf5Reader.Get(*var_r64, R64.data()); + + hdf5Reader.EndStep(); + + EXPECT_EQ(IString, currentTestData.S1); + + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + hdf5Reader.Close(); + } +} + +//****************************************************************************** +// 2D 2x4 test data +//****************************************************************************** + +// ADIOS2 write, native HDF5 read +TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D2x4) +{ + // Each process would write a 2x4 array and all processes would + // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here + std::string fname = "ADIOS2HDF5WriteHDF5Read2D2x4Test.h5"; + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 4; + + // Number of rows + const std::size_t Ny = 2; + + // Number of steps + const std::size_t NSteps = 3; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + + // Write test data using ADIOS2 + { +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + adios2::IO &io = adios.DeclareIO("TestIO"); + + // Declare 2D variables (Ny * (NumOfProcesses * Nx)) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + adios2::Dims shape{static_cast(Ny), + static_cast(Nx * mpiSize)}; + adios2::Dims start{static_cast(0), + static_cast(mpiRank * Nx)}; + adios2::Dims count{static_cast(Ny), + static_cast(Nx)}; + auto &var_iString = io.DefineVariable("iString"); + auto &var_i8 = io.DefineVariable("i8", shape, start, count); + auto &var_i16 = + io.DefineVariable("i16", shape, start, count); + auto &var_i32 = + io.DefineVariable("i32", shape, start, count); + auto &var_i64 = + io.DefineVariable("i64", shape, start, count); + auto &var_u8 = + io.DefineVariable("u8", shape, start, count); + auto &var_u16 = + io.DefineVariable("u16", shape, start, count); + auto &var_u32 = + io.DefineVariable("u32", shape, start, count); + auto &var_u64 = + io.DefineVariable("u64", shape, start, count); + auto &var_r32 = + io.DefineVariable("r32", shape, start, count); + auto &var_r64 = + io.DefineVariable("r64", shape, start, count); + } + + // Create the HDF5 Engine + io.SetEngine("HDF5"); + + // HDf5 engine calls the HDF5 common object that calls the hDF5 library. + // The IO functionality, SetParameters and AddTransports will be added + // in the future. For now `io.AddTransport("file", { + // "library", "MPI"}});` is omitted. + // }) + io.AddTransport("file"); + + adios2::Engine &engine = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + auto &var_iString = *io.InquireVariable("iString"); + auto &var_i8 = *io.InquireVariable("i8"); + auto &var_i16 = *io.InquireVariable("i16"); + auto &var_i32 = *io.InquireVariable("i32"); + auto &var_i64 = *io.InquireVariable("i64"); + auto &var_u8 = *io.InquireVariable("u8"); + auto &var_u16 = *io.InquireVariable("u16"); + auto &var_u32 = *io.InquireVariable("u32"); + auto &var_u64 = *io.InquireVariable("u64"); + auto &var_r32 = *io.InquireVariable("r32"); + auto &var_r64 = *io.InquireVariable("r64"); + + // Make a 2D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel( + {0, static_cast(mpiRank * Nx)}, {Ny, Nx}); + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + engine.Put(var_iString, currentTestData.S1); + engine.Put(var_i8, currentTestData.I8.data()); + engine.Put(var_i16, currentTestData.I16.data()); + engine.Put(var_i32, currentTestData.I32.data()); + engine.Put(var_i64, currentTestData.I64.data()); + engine.Put(var_u8, currentTestData.U8.data()); + engine.Put(var_u16, currentTestData.U16.data()); + engine.Put(var_u32, currentTestData.U32.data()); + engine.Put(var_u64, currentTestData.U64.data()); + engine.Put(var_r32, currentTestData.R32.data()); + engine.Put(var_r64, currentTestData.R64.data()); + + // Advance to the next time step + engine.EndStep(); + } + + // Close the file + engine.Close(); + } + + { + HDF5NativeReader hdf5Reader(fname); + + std::string IString; + const size_t arraySize = Nx * Ny; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + // 2D + hsize_t count[2], offset[2]; + + offset[0] = 0; + offset[1] = mpiRank * Nx; + count[0] = Ny; + count[1] = Nx; + + size_t globalArraySize = Nx * mpiSize; + + // For each variable, we would verify its global size and type. + // Then we would retrieve the data back which is written by the + // current process and validate the value + for (size_t t = 0; t < NSteps; ++t) + { + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); + + std::vector gDims; + hid_t h5Type; + + hdf5Reader.GetVarInfo("iString", gDims, h5Type); + // ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_IN), 1); + ASSERT_EQ(gDims.size(), 0); + hdf5Reader.ReadString("iString", IString); + + hdf5Reader.GetVarInfo("i8", gDims, h5Type); + ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_INT8), 1); + ASSERT_EQ(gDims.size(), 2); + ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[1], globalArraySize); + hdf5Reader.ReadVar("i8", I8.data(), offset, count, arraySize); + + hdf5Reader.GetVarInfo("i16", gDims, h5Type); + ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_SHORT), 1); + ASSERT_EQ(gDims.size(), 2); + ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[1], globalArraySize); + hdf5Reader.ReadVar("i16", I16.data(), offset, count, arraySize); + + hdf5Reader.GetVarInfo("i32", gDims, h5Type); + ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_INT), 1); + ASSERT_EQ(gDims.size(), 2); + ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[1], globalArraySize); + hdf5Reader.ReadVar("i32", I32.data(), offset, count, arraySize); + + hdf5Reader.GetVarInfo("i64", gDims, h5Type); + ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_LONG), 1); + ASSERT_EQ(gDims.size(), 2); + ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[1], globalArraySize); + hdf5Reader.ReadVar("i64", I64.data(), offset, count, arraySize); + + hdf5Reader.GetVarInfo("u8", gDims, h5Type); + ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_UCHAR), 1); + ASSERT_EQ(gDims.size(), 2); + ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[1], globalArraySize); + hdf5Reader.ReadVar("u8", U8.data(), offset, count, arraySize); + + hdf5Reader.GetVarInfo("u16", gDims, h5Type); + ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_USHORT), 1); + ASSERT_EQ(gDims.size(), 2); + ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[1], globalArraySize); + hdf5Reader.ReadVar("u16", U16.data(), offset, count, arraySize); + + hdf5Reader.GetVarInfo("u32", gDims, h5Type); + ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_UINT), 1); + ASSERT_EQ(gDims.size(), 2); + ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[1], globalArraySize); + hdf5Reader.ReadVar("u32", U32.data(), offset, count, arraySize); + + hdf5Reader.GetVarInfo("u64", gDims, h5Type); + ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_ULONG), 1); + ASSERT_EQ(gDims.size(), 2); + ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[1], globalArraySize); + hdf5Reader.ReadVar("u64", U64.data(), offset, count, arraySize); + + hdf5Reader.GetVarInfo("r32", gDims, h5Type); + ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_FLOAT), 1); + ASSERT_EQ(gDims.size(), 2); + ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[1], globalArraySize); + hdf5Reader.ReadVar("r32", R32.data(), offset, count, arraySize); + + hdf5Reader.GetVarInfo("r64", gDims, h5Type); + ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_DOUBLE), 1); + ASSERT_EQ(gDims.size(), 2); + ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[1], globalArraySize); + hdf5Reader.ReadVar("r64", R64.data(), offset, count, arraySize); + + EXPECT_EQ(IString, currentTestData.S1); + + // Check if it's correct + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + hdf5Reader.Advance(); + } + } +} + +// ADIOS2 write, ADIOS2 read +TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteADIOS2HDF5Read2D2x4) +{ + std::string fname = "ADIOS2HDF5WriteADIOS2HDF5Read2D2x4Test.h5"; + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 4; + + // Number of rows + const std::size_t Ny = 2; + + // Number of steps + const std::size_t NSteps = 3; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + // Write test data using ADIOS2 + { + adios2::IO &io = adios.DeclareIO("TestIO"); + + // Declare 2D variables (Ny * (NumOfProcesses * Nx)) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + adios2::Dims shape{static_cast(Ny), + static_cast(Nx * mpiSize)}; + adios2::Dims start{static_cast(0), + static_cast(mpiRank * Nx)}; + adios2::Dims count{static_cast(Ny), + static_cast(Nx)}; + + auto &var_iString = io.DefineVariable("iString"); + + auto &var_i8 = io.DefineVariable("i8", shape, start, count); + auto &var_i16 = + io.DefineVariable("i16", shape, start, count); + auto &var_i32 = + io.DefineVariable("i32", shape, start, count); + auto &var_i64 = + io.DefineVariable("i64", shape, start, count); + auto &var_u8 = + io.DefineVariable("u8", shape, start, count); + auto &var_u16 = + io.DefineVariable("u16", shape, start, count); + auto &var_u32 = + io.DefineVariable("u32", shape, start, count); + auto &var_u64 = + io.DefineVariable("u64", shape, start, count); + auto &var_r32 = + io.DefineVariable("r32", shape, start, count); + auto &var_r64 = + io.DefineVariable("r64", shape, start, count); + } + + // Create the HDF5 Engine + io.SetEngine("HDF5"); + + // HDf5 engine calls the HDF5 common object that calls the hDF5 library. + // The IO functionality, SetParameters and AddTransports will be added + // in the future. For now `io.AddTransport("file", { + // "library", "MPI"}});` is omitted. + // }) + io.AddTransport("file"); + + adios2::Engine &engine = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + auto &var_iString = *io.InquireVariable("iString"); + auto &var_i8 = *io.InquireVariable("i8"); + auto &var_i16 = *io.InquireVariable("i16"); + auto &var_i32 = *io.InquireVariable("i32"); + auto &var_i64 = *io.InquireVariable("i64"); + auto &var_u8 = *io.InquireVariable("u8"); + auto &var_u16 = *io.InquireVariable("u16"); + auto &var_u32 = *io.InquireVariable("u32"); + auto &var_u64 = *io.InquireVariable("u64"); + auto &var_r32 = *io.InquireVariable("r32"); + auto &var_r64 = *io.InquireVariable("r64"); + + // Make a 2D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel( + {0, static_cast(mpiRank * Nx)}, {Ny, Nx}); + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + engine.Put(var_iString, currentTestData.S1); + engine.Put(var_i8, currentTestData.I8.data()); + engine.Put(var_i16, currentTestData.I16.data()); + engine.Put(var_i32, currentTestData.I32.data()); + engine.Put(var_i64, currentTestData.I64.data()); + engine.Put(var_u8, currentTestData.U8.data()); + engine.Put(var_u16, currentTestData.U16.data()); + engine.Put(var_u32, currentTestData.U32.data()); + engine.Put(var_u64, currentTestData.U64.data()); + engine.Put(var_r32, currentTestData.R32.data()); + engine.Put(var_r64, currentTestData.R64.data()); + + // Advance to the next time step + engine.EndStep(); + } + + // Close the file + engine.Close(); + } + + { // reading back + adios2::IO &io = adios.DeclareIO("HDF5ReadIO"); + io.SetEngine("HDF5"); + + adios2::Engine &hdf5Reader = io.Open(fname, adios2::Mode::Read); + + auto var_iString = io.InquireVariable("iString"); + ASSERT_NE(var_iString, nullptr); + ASSERT_EQ(var_iString->m_Shape.size(), 0); + ASSERT_EQ(var_iString->m_AvailableStepsCount, NSteps); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], Ny); + ASSERT_EQ(var_i8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], Ny); + ASSERT_EQ(var_i16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], Ny); + ASSERT_EQ(var_i32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], Ny); + ASSERT_EQ(var_i64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], Ny); + ASSERT_EQ(var_u8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], Ny); + ASSERT_EQ(var_u16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], Ny); + ASSERT_EQ(var_u32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], Ny); + ASSERT_EQ(var_u64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], Ny); + ASSERT_EQ(var_r32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], Ny); + ASSERT_EQ(var_r64->m_Shape[1], static_cast(mpiSize * Nx)); + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + for (size_t t = 0; t < NSteps; ++t) + { + var_i8->SetStepSelection({t, 1}); + var_i16->SetStepSelection({t, 1}); + var_i32->SetStepSelection({t, 1}); + var_i64->SetStepSelection({t, 1}); + + var_u8->SetStepSelection({t, 1}); + var_u16->SetStepSelection({t, 1}); + var_u32->SetStepSelection({t, 1}); + var_u64->SetStepSelection({t, 1}); + + var_r32->SetStepSelection({t, 1}); + var_r64->SetStepSelection({t, 1}); + + hdf5Reader.Get(*var_iString, IString); + + hdf5Reader.Get(*var_i8, I8.data()); + hdf5Reader.Get(*var_i16, I16.data()); + hdf5Reader.Get(*var_i32, I32.data()); + hdf5Reader.Get(*var_i64, I64.data()); + + hdf5Reader.Get(*var_u8, U8.data()); + hdf5Reader.Get(*var_u16, U16.data()); + hdf5Reader.Get(*var_u32, U32.data()); + hdf5Reader.Get(*var_u64, U64.data()); + + hdf5Reader.Get(*var_r32, R32.data()); + hdf5Reader.Get(*var_r64, R64.data()); + + hdf5Reader.PerformGets(); + + // Generate test data for each rank uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(t), mpiRank, mpiSize); + + EXPECT_EQ(IString, currentTestData.S1); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + hdf5Reader.Close(); + } +} + +// Native HDF5 write, ADIOS2 read +TEST_F(HDF5WriteReadTest, HDF5WriteADIOS2HDF5Read2D2x4) +{ + std::string fname = "HDF5WriteADIOS2HDF5Read2D2x4Test.h5"; + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 4; + const std::size_t Ny = 2; + // Number of steps + const std::size_t NSteps = 3; + + { +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + + HDF5NativeWriter h5writer(fname, MPI_COMM_WORLD); +#else + HDF5NativeWriter h5writer(fname, 0); +#endif + + int dimSize = 2; + hsize_t global_dims[2] = {Ny, Nx * mpiSize}; + hsize_t count[2] = {Ny, Nx}; + hsize_t offset[2] = {0, Nx * mpiRank}; + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + + h5writer.CreateAndStoreScalar("iString", H5T_STRING, + currentTestData.S1.data()); + h5writer.CreateAndStoreVar("i8", dimSize, H5T_NATIVE_INT8, + global_dims, offset, count, + currentTestData.I8.data()); + h5writer.CreateAndStoreVar("i16", dimSize, H5T_NATIVE_SHORT, + global_dims, offset, count, + currentTestData.I16.data()); + h5writer.CreateAndStoreVar("i32", dimSize, H5T_NATIVE_INT, + global_dims, offset, count, + currentTestData.I32.data()); + h5writer.CreateAndStoreVar("i64", dimSize, H5T_NATIVE_LONG, + global_dims, offset, count, + currentTestData.I64.data()); + h5writer.CreateAndStoreVar("u8", dimSize, H5T_NATIVE_UCHAR, + global_dims, offset, count, + currentTestData.U8.data()); + h5writer.CreateAndStoreVar("u16", dimSize, H5T_NATIVE_USHORT, + global_dims, offset, count, + currentTestData.U16.data()); + h5writer.CreateAndStoreVar("u32", dimSize, H5T_NATIVE_UINT, + global_dims, offset, count, + currentTestData.U32.data()); + h5writer.CreateAndStoreVar("u64", dimSize, H5T_NATIVE_ULONG, + global_dims, offset, count, + currentTestData.U64.data()); + h5writer.CreateAndStoreVar("r32", dimSize, H5T_NATIVE_FLOAT, + global_dims, offset, count, + currentTestData.R32.data()); + h5writer.CreateAndStoreVar("r64", dimSize, H5T_NATIVE_DOUBLE, + global_dims, offset, count, + currentTestData.R64.data()); + h5writer.Advance(); + } + } + + { // read back +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + adios2::IO &io = adios.DeclareIO("HDF5ReadIO"); + io.SetEngine("HDF5"); + + adios2::Engine &hdf5Reader = io.Open(fname, adios2::Mode::Read); + + auto var_iString = io.InquireVariable("iString"); + ASSERT_NE(var_iString, nullptr); + ASSERT_EQ(var_iString->m_Shape.size(), 0); + ASSERT_EQ(var_iString->m_AvailableStepsCount, NSteps); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], Ny); + ASSERT_EQ(var_i8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], Ny); + ASSERT_EQ(var_i16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], Ny); + ASSERT_EQ(var_i32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], Ny); + ASSERT_EQ(var_i64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], Ny); + ASSERT_EQ(var_u8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], Ny); + ASSERT_EQ(var_u16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], Ny); + ASSERT_EQ(var_u32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], Ny); + ASSERT_EQ(var_u64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], Ny); + ASSERT_EQ(var_r32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], Ny); + ASSERT_EQ(var_r64->m_Shape[1], static_cast(mpiSize * Nx)); + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + for (size_t t = 0; t < NSteps; ++t) + { + var_i8->SetStepSelection({t, 1}); + var_i16->SetStepSelection({t, 1}); + var_i32->SetStepSelection({t, 1}); + var_i64->SetStepSelection({t, 1}); + + var_u8->SetStepSelection({t, 1}); + var_u16->SetStepSelection({t, 1}); + var_u32->SetStepSelection({t, 1}); + var_u64->SetStepSelection({t, 1}); + + var_r32->SetStepSelection({t, 1}); + var_r64->SetStepSelection({t, 1}); + + hdf5Reader.Get(*var_iString, IString); + + hdf5Reader.Get(*var_i8, I8.data()); + hdf5Reader.Get(*var_i16, I16.data()); + hdf5Reader.Get(*var_i32, I32.data()); + hdf5Reader.Get(*var_i64, I64.data()); + + hdf5Reader.Get(*var_u8, U8.data()); + hdf5Reader.Get(*var_u16, U16.data()); + hdf5Reader.Get(*var_u32, U32.data()); + hdf5Reader.Get(*var_u64, U64.data()); + + hdf5Reader.Get(*var_r32, R32.data()); + hdf5Reader.Get(*var_r64, R64.data()); + + hdf5Reader.PerformGets(); + + // Generate test data for each rank uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(t), mpiRank, mpiSize); + + EXPECT_EQ(IString, currentTestData.S1); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + hdf5Reader.Close(); + } +} + +//****************************************************************************** +// 2D 4x2 test data +//****************************************************************************** + +// ADIOS2 write, native HDF5 read +TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D4x2) +{ + + // Each process would write a 4x2 array and all processes would + // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here + std::string fname = "ADIOS2HDF5WriteHDF5Read2D4x2Test.h5"; + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 2; + // Number of cols + const std::size_t Ny = 4; + + // Number of steps + const std::size_t NSteps = 3; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + + // Write test data using ADIOS2 + { +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + adios2::IO &io = adios.DeclareIO("TestIO"); + + // Declare 2D variables (4 * (NumberOfProcess * Nx)) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + adios2::Dims shape{static_cast(Ny), + static_cast(mpiSize * Nx)}; + adios2::Dims start{static_cast(0), + static_cast(mpiRank * Nx)}; + adios2::Dims count{static_cast(Ny), + static_cast(Nx)}; + + auto &var_iString = io.DefineVariable("iString"); + auto &var_i8 = io.DefineVariable("i8", shape, start, count); + auto &var_i16 = + io.DefineVariable("i16", shape, start, count); + auto &var_i32 = + io.DefineVariable("i32", shape, start, count); + auto &var_i64 = + io.DefineVariable("i64", shape, start, count); + auto &var_u8 = + io.DefineVariable("u8", shape, start, count); + auto &var_u16 = + io.DefineVariable("u16", shape, start, count); + auto &var_u32 = + io.DefineVariable("u32", shape, start, count); + auto &var_u64 = + io.DefineVariable("u64", shape, start, count); + auto &var_r32 = + io.DefineVariable("r32", shape, start, count); + auto &var_r64 = + io.DefineVariable("r64", shape, start, count); + } + + // Create the HDF5 Engine + io.SetEngine("HDF5"); + + // HDf5 engine calls the HDF5 common object that calls the hDF5 library. + // The IO functionality, SetParameters and AddTransports will be added + // in the future. For now `io.AddTransport("file", { + // "library", "MPI"}});` is omitted. + // }) + io.AddTransport("file"); + + adios2::Engine &engine = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + auto &var_iString = *io.InquireVariable("iString"); + auto &var_i8 = *io.InquireVariable("i8"); + auto &var_i16 = *io.InquireVariable("i16"); + auto &var_i32 = *io.InquireVariable("i32"); + auto &var_i64 = *io.InquireVariable("i64"); + auto &var_u8 = *io.InquireVariable("u8"); + auto &var_u16 = *io.InquireVariable("u16"); + auto &var_u32 = *io.InquireVariable("u32"); + auto &var_u64 = *io.InquireVariable("u64"); + auto &var_r32 = *io.InquireVariable("r32"); + auto &var_r64 = *io.InquireVariable("r64"); + + // Make a 2D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces adios2::Box sel( {0, static_cast(mpiRank * Nx)}, {Ny, Nx}); var_i8.SetSelection(sel); @@ -577,6 +2183,8 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D2x4) // Write each one // fill in the variable with values from starting index to // starting index + count + engine.BeginStep(); + engine.Put(var_iString, currentTestData.S1); engine.Put(var_i8, currentTestData.I8.data()); engine.Put(var_i16, currentTestData.I16.data()); engine.Put(var_i32, currentTestData.I32.data()); @@ -587,18 +2195,17 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D2x4) engine.Put(var_u64, currentTestData.U64.data()); engine.Put(var_r32, currentTestData.R32.data()); engine.Put(var_r64, currentTestData.R64.data()); - - // Advance to the next time step engine.EndStep(); } - // Close the file engine.Close(); } { + HDF5NativeReader hdf5Reader(fname); + std::string IString; const size_t arraySize = Nx * Ny; std::array I8; std::array I16; @@ -612,10 +2219,10 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D2x4) std::array R64; // 2D hsize_t count[2], offset[2]; - count[0] = 0; - count[1] = mpiRank * Nx; - offset[0] = Ny; - offset[1] = Nx; + offset[0] = 0; + offset[1] = mpiRank * Nx; + count[0] = Ny; + count[1] = Nx; size_t globalArraySize = Nx * mpiSize; // For each variable, we would verify its global size and type. @@ -629,76 +2236,82 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D2x4) std::vector gDims; hid_t h5Type; + hdf5Reader.GetVarInfo("iString", gDims, h5Type); + // ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_IN), 1); + ASSERT_EQ(gDims.size(), 0); + hdf5Reader.ReadString("iString", IString); + hdf5Reader.GetVarInfo("i8", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_INT8), 1); ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[0], 4); ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("i8", I8.data(), count, offset, arraySize); + hdf5Reader.ReadVar("i8", I8.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("i16", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_SHORT), 1); ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[0], 4); ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("i16", I16.data(), count, offset, arraySize); + hdf5Reader.ReadVar("i16", I16.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("i32", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_INT), 1); ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[0], 4); ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("i32", I32.data(), count, offset, arraySize); + hdf5Reader.ReadVar("i32", I32.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("i64", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_LONG), 1); ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[0], 4); ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("i64", I64.data(), count, offset, arraySize); + hdf5Reader.ReadVar("i64", I64.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("u8", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_UCHAR), 1); ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[0], 4); ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("u8", U8.data(), count, offset, arraySize); + hdf5Reader.ReadVar("u8", U8.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("u16", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_USHORT), 1); ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[0], 4); ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("u16", U16.data(), count, offset, arraySize); + hdf5Reader.ReadVar("u16", U16.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("u32", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_UINT), 1); ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[0], 4); ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("u32", U32.data(), count, offset, arraySize); + hdf5Reader.ReadVar("u32", U32.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("u64", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_ULONG), 1); ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[0], 4); ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("u64", U64.data(), count, offset, arraySize); + hdf5Reader.ReadVar("u64", U64.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("r32", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_FLOAT), 1); ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[0], 4); ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("r32", R32.data(), count, offset, arraySize); + hdf5Reader.ReadVar("r32", R32.data(), offset, count, arraySize); hdf5Reader.GetVarInfo("r64", gDims, h5Type); ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_DOUBLE), 1); ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 2); + ASSERT_EQ(gDims[0], 4); ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("r64", R64.data(), count, offset, arraySize); + hdf5Reader.ReadVar("r64", R64.data(), offset, count, arraySize); + EXPECT_EQ(IString, currentTestData.S1); // Check if it's correct for (size_t i = 0; i < Nx; ++i) { @@ -723,32 +2336,9 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D2x4) } // ADIOS2 write, ADIOS2 read -TEST_F(HDF5WriteReadTest, DISABLED_ADIOS2HDF5WriteADIOS2HDF5Read2D2x4) -{ - // std::string fname = "ADIOS2HDF5WriteADIOS2HDF5Read2D2x4Test.h5"; - - ASSERT_TRUE(false) << "ADIOS2 read API is not yet implemented"; -} - -// Native HDF5 write, ADIOS2 read -TEST_F(HDF5WriteReadTest, DISABLED_HDF5WriteADIOS2HDF5Read2D2x4) -{ - // std::string fname = "HDF5WriteADIOS2HDF5Read2D2x4Test.h5"; - - ASSERT_TRUE(false) << "ADIOS2 read API is not yet implemented"; -} - -//****************************************************************************** -// 2D 4x2 test data -//****************************************************************************** - -// ADIOS2 write, native HDF5 read -TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D4x2) +TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteADIOS2HDF5Read2D4x2) { - - // Each process would write a 4x2 array and all processes would - // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - std::string fname = "ADIOS2HDF5WriteHDF5Read2D4x2Test.h5"; + std::string fname = "ADIOS2HDF5WriteADIOS2HDF5Read2D4x2Test.h5"; int mpiRank = 0, mpiSize = 1; // Number of rows @@ -764,13 +2354,14 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D4x2) MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); #endif - // Write test data using ADIOS2 - { #ifdef ADIOS2_HAVE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); #else - adios2::ADIOS adios(true); + adios2::ADIOS adios(true); #endif + + // Write test data using ADIOS2 + { adios2::IO &io = adios.DeclareIO("TestIO"); // Declare 2D variables (4 * (NumberOfProcess * Nx)) @@ -783,6 +2374,7 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D4x2) static_cast(mpiRank * Nx)}; adios2::Dims count{static_cast(Ny), static_cast(Nx)}; + auto &var_iString = io.DefineVariable("iString"); auto &var_i8 = io.DefineVariable("i8", shape, start, count); auto &var_i16 = io.DefineVariable("i16", shape, start, count); @@ -823,6 +2415,7 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D4x2) generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); // Retrieve the variables that previously went out of scope + auto &var_iString = *io.InquireVariable("iString"); auto &var_i8 = *io.InquireVariable("i8"); auto &var_i16 = *io.InquireVariable("i16"); auto &var_i32 = *io.InquireVariable("i32"); @@ -853,6 +2446,7 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D4x2) // fill in the variable with values from starting index to // starting index + count engine.BeginStep(); + engine.Put(var_iString, currentTestData.S1); engine.Put(var_i8, currentTestData.I8.data()); engine.Put(var_i16, currentTestData.I16.data()); engine.Put(var_i32, currentTestData.I32.data()); @@ -869,112 +2463,157 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D4x2) engine.Close(); } - { - - HDF5NativeReader hdf5Reader(fname); + { // reading back + adios2::IO &io = adios.DeclareIO("HDF5ReadIO"); + io.SetEngine("HDF5"); - const size_t arraySize = Nx * Ny; - std::array I8; - std::array I16; - std::array I32; - std::array I64; - std::array U8; - std::array U16; - std::array U32; - std::array U64; - std::array R32; - std::array R64; - // 2D - hsize_t count[2], offset[2]; - count[0] = 0; - count[1] = mpiRank * Nx; - offset[0] = Ny; - offset[1] = Nx; - size_t globalArraySize = Nx * mpiSize; + adios2::Engine &hdf5Reader = io.Open(fname, adios2::Mode::Read); + + auto var_iString = io.InquireVariable("iString"); + ASSERT_NE(var_iString, nullptr); + ASSERT_EQ(var_iString->m_Shape.size(), 0); + ASSERT_EQ(var_iString->m_AvailableStepsCount, NSteps); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], Ny); + ASSERT_EQ(var_i8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], Ny); + ASSERT_EQ(var_i16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], Ny); + ASSERT_EQ(var_i32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], Ny); + ASSERT_EQ(var_i64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], Ny); + ASSERT_EQ(var_u8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], Ny); + ASSERT_EQ(var_u16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], Ny); + ASSERT_EQ(var_u32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], Ny); + ASSERT_EQ(var_u64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], Ny); + ASSERT_EQ(var_r32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], Ny); + ASSERT_EQ(var_r64->m_Shape[1], static_cast(mpiSize * Nx)); + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); - // For each variable, we would verify its global size and type. - // Then we would retrieve the data back which is written by the - // current process and validate the value for (size_t t = 0; t < NSteps; ++t) { - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); - - std::vector gDims; - hid_t h5Type; - - hdf5Reader.GetVarInfo("i8", gDims, h5Type); - ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_INT8), 1); - ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 4); - ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("i8", I8.data(), count, offset, arraySize); + var_i8->SetStepSelection({t, 1}); + var_i16->SetStepSelection({t, 1}); + var_i32->SetStepSelection({t, 1}); + var_i64->SetStepSelection({t, 1}); - hdf5Reader.GetVarInfo("i16", gDims, h5Type); - ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_SHORT), 1); - ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 4); - ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("i16", I16.data(), count, offset, arraySize); + var_u8->SetStepSelection({t, 1}); + var_u16->SetStepSelection({t, 1}); + var_u32->SetStepSelection({t, 1}); + var_u64->SetStepSelection({t, 1}); - hdf5Reader.GetVarInfo("i32", gDims, h5Type); - ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_INT), 1); - ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 4); - ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("i32", I32.data(), count, offset, arraySize); + var_r32->SetStepSelection({t, 1}); + var_r64->SetStepSelection({t, 1}); - hdf5Reader.GetVarInfo("i64", gDims, h5Type); - ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_LONG), 1); - ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 4); - ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("i64", I64.data(), count, offset, arraySize); + hdf5Reader.BeginStep(); + hdf5Reader.Get(*var_iString, IString); - hdf5Reader.GetVarInfo("u8", gDims, h5Type); - ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_UCHAR), 1); - ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 4); - ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("u8", U8.data(), count, offset, arraySize); + hdf5Reader.Get(*var_i8, I8.data()); + hdf5Reader.Get(*var_i16, I16.data()); + hdf5Reader.Get(*var_i32, I32.data()); + hdf5Reader.Get(*var_i64, I64.data()); - hdf5Reader.GetVarInfo("u16", gDims, h5Type); - ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_USHORT), 1); - ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 4); - ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("u16", U16.data(), count, offset, arraySize); + hdf5Reader.Get(*var_u8, U8.data()); + hdf5Reader.Get(*var_u16, U16.data()); + hdf5Reader.Get(*var_u32, U32.data()); + hdf5Reader.Get(*var_u64, U64.data()); - hdf5Reader.GetVarInfo("u32", gDims, h5Type); - ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_UINT), 1); - ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 4); - ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("u32", U32.data(), count, offset, arraySize); + hdf5Reader.Get(*var_r32, R32.data()); + hdf5Reader.Get(*var_r64, R64.data()); - hdf5Reader.GetVarInfo("u64", gDims, h5Type); - ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_ULONG), 1); - ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 4); - ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("u64", U64.data(), count, offset, arraySize); + hdf5Reader.EndStep(); - hdf5Reader.GetVarInfo("r32", gDims, h5Type); - ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_FLOAT), 1); - ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 4); - ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("r32", R32.data(), count, offset, arraySize); + // Generate test data for each rank uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(t), mpiRank, mpiSize); - hdf5Reader.GetVarInfo("r64", gDims, h5Type); - ASSERT_EQ(H5Tequal(h5Type, H5T_NATIVE_DOUBLE), 1); - ASSERT_EQ(gDims.size(), 2); - ASSERT_EQ(gDims[0], 4); - ASSERT_EQ(gDims[1], globalArraySize); - hdf5Reader.ReadVar("r64", R64.data(), count, offset, arraySize); + EXPECT_EQ(IString, currentTestData.S1); - // Check if it's correct - for (size_t i = 0; i < Nx; ++i) + for (size_t i = 0; i < Nx * Ny; ++i) { std::stringstream ss; ss << "t=" << t << " i=" << i << " rank=" << mpiRank; @@ -991,25 +2630,260 @@ TEST_F(HDF5WriteReadTest, ADIOS2HDF5WriteHDF5Read2D4x2) EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; } - hdf5Reader.Advance(); } + hdf5Reader.Close(); } } -// ADIOS2 write, ADIOS2 read -TEST_F(HDF5WriteReadTest, DISABLED_ADIOS2HDF5WriteADIOS2HDF5Read2D4x2) +// Native HDF5 write, ADIOS2 read +TEST_F(HDF5WriteReadTest, HDF5WriteADIOS2HDF5Read2D4x2) { - // std::string fname = "ADIOS2HDF5WriteADIOS2HDF5Read2D4x2Test.h5"; + std::string fname = "HDF5WriteADIOS2HDF5Read2D4x2Test.h5"; - ASSERT_TRUE(false) << "ADIOS2 read API is not yet implemented"; -} + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 2; + const std::size_t Ny = 4; + // Number of steps + const std::size_t NSteps = 3; -// Native HDF5 write, ADIOS2 read -TEST_F(HDF5WriteReadTest, DISABLED_HDF5WriteADIOS2HDF5Read2D4x2) -{ - // std::string fname = "HDF5WriteADIOS2HDF5Read2D4x2Test.h5"; + { +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + + HDF5NativeWriter h5writer(fname, MPI_COMM_WORLD); +#else + HDF5NativeWriter h5writer(fname, 0); +#endif + + int dimSize = 2; + hsize_t global_dims[2] = {Ny, Nx * mpiSize}; + hsize_t count[2] = {Ny, Nx}; + hsize_t offset[2] = {0, Nx * mpiRank}; + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); + + h5writer.CreateAndStoreScalar("iString", H5T_STRING, + currentTestData.S1.data()); + h5writer.CreateAndStoreVar("i8", dimSize, H5T_NATIVE_INT8, + global_dims, offset, count, + currentTestData.I8.data()); + h5writer.CreateAndStoreVar("i16", dimSize, H5T_NATIVE_SHORT, + global_dims, offset, count, + currentTestData.I16.data()); + h5writer.CreateAndStoreVar("i32", dimSize, H5T_NATIVE_INT, + global_dims, offset, count, + currentTestData.I32.data()); + h5writer.CreateAndStoreVar("i64", dimSize, H5T_NATIVE_LONG, + global_dims, offset, count, + currentTestData.I64.data()); + h5writer.CreateAndStoreVar("u8", dimSize, H5T_NATIVE_UCHAR, + global_dims, offset, count, + currentTestData.U8.data()); + h5writer.CreateAndStoreVar("u16", dimSize, H5T_NATIVE_USHORT, + global_dims, offset, count, + currentTestData.U16.data()); + h5writer.CreateAndStoreVar("u32", dimSize, H5T_NATIVE_UINT, + global_dims, offset, count, + currentTestData.U32.data()); + h5writer.CreateAndStoreVar("u64", dimSize, H5T_NATIVE_ULONG, + global_dims, offset, count, + currentTestData.U64.data()); + h5writer.CreateAndStoreVar("r32", dimSize, H5T_NATIVE_FLOAT, + global_dims, offset, count, + currentTestData.R32.data()); + h5writer.CreateAndStoreVar("r64", dimSize, H5T_NATIVE_DOUBLE, + global_dims, offset, count, + currentTestData.R64.data()); + h5writer.Advance(); + } + } + + { // read back +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif - ASSERT_TRUE(false) << "ADIOS2 read API is not yet implemented"; + adios2::IO &io = adios.DeclareIO("HDF5ReadIO"); + io.SetEngine("HDF5"); + + adios2::Engine &hdf5Reader = io.Open(fname, adios2::Mode::Read); + + auto var_iString = io.InquireVariable("iString"); + ASSERT_NE(var_iString, nullptr); + ASSERT_EQ(var_iString->m_Shape.size(), 0); + ASSERT_EQ(var_iString->m_AvailableStepsCount, NSteps); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], Ny); + ASSERT_EQ(var_i8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], Ny); + ASSERT_EQ(var_i16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], Ny); + ASSERT_EQ(var_i32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], Ny); + ASSERT_EQ(var_i64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], Ny); + ASSERT_EQ(var_u8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], Ny); + ASSERT_EQ(var_u16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], Ny); + ASSERT_EQ(var_u32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], Ny); + ASSERT_EQ(var_u64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], Ny); + ASSERT_EQ(var_r32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], Ny); + ASSERT_EQ(var_r64->m_Shape[1], static_cast(mpiSize * Nx)); + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + for (size_t t = 0; t < NSteps; ++t) + { + var_i8->SetStepSelection({t, 1}); + var_i16->SetStepSelection({t, 1}); + var_i32->SetStepSelection({t, 1}); + var_i64->SetStepSelection({t, 1}); + + var_u8->SetStepSelection({t, 1}); + var_u16->SetStepSelection({t, 1}); + var_u32->SetStepSelection({t, 1}); + var_u64->SetStepSelection({t, 1}); + + var_r32->SetStepSelection({t, 1}); + var_r64->SetStepSelection({t, 1}); + + hdf5Reader.BeginStep(); + hdf5Reader.Get(*var_iString, IString); + + hdf5Reader.Get(*var_i8, I8.data()); + hdf5Reader.Get(*var_i16, I16.data()); + hdf5Reader.Get(*var_i32, I32.data()); + hdf5Reader.Get(*var_i64, I64.data()); + + hdf5Reader.Get(*var_u8, U8.data()); + hdf5Reader.Get(*var_u16, U16.data()); + hdf5Reader.Get(*var_u32, U32.data()); + hdf5Reader.Get(*var_u64, U64.data()); + + hdf5Reader.Get(*var_r32, R32.data()); + hdf5Reader.Get(*var_r64, R64.data()); + hdf5Reader.EndStep(); + + // Generate test data for each rank uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(t), mpiRank, mpiSize); + + EXPECT_EQ(IString, currentTestData.S1); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + hdf5Reader.Close(); + } } //******************************************************************************