From 20adab597012e788f9f641f3c56d5904fea3136a Mon Sep 17 00:00:00 2001 From: pradsubedi Date: Tue, 1 Oct 2019 22:04:57 -0400 Subject: [PATCH 1/3] Added support for Scalar Variable --- .../engine/dataspaces/DataSpacesReader.cpp | 19 ++++--- .../engine/dataspaces/DataSpacesReader.tcc | 47 +++++++++------- .../engine/dataspaces/DataSpacesWriter.cpp | 21 ++++---- .../engine/dataspaces/DataSpacesWriter.tcc | 54 +++++++++++-------- 4 files changed, 80 insertions(+), 61 deletions(-) diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.cpp b/source/adios2/engine/dataspaces/DataSpacesReader.cpp index 79c698f00e..61e0c8d949 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesReader.cpp @@ -81,27 +81,24 @@ StepStatus DataSpacesReader::BeginStep(StepMode mode, const float timeout_sec) meta_lk = new char[lk_name.length() + 1]; strcpy(meta_lk, lk_name.c_str()); - MPI_Comm lock_comm = MPI_COMM_SELF; - + MPI_Comm lock_comm = m_data.mpi_comm; + dspaces_lock_on_read(meta_lk, &lock_comm); + int nVars = 0; if (!m_ProvideLatest) { if (rank == 0) { - dspaces_lock_on_read(meta_lk, &lock_comm); buffer = dspaces_get_next_meta(m_CurrentStep, fstr, &bcast_array[0], &bcast_array[1]); - dspaces_unlock_on_read(meta_lk, &lock_comm); } } else { if (rank == 0) { - dspaces_lock_on_read(meta_lk, &lock_comm); buffer = dspaces_get_latest_meta(m_CurrentStep, fstr, &bcast_array[0], &bcast_array[1]); - dspaces_unlock_on_read(meta_lk, &lock_comm); } } MPI_Bcast(bcast_array, 2, MPI_INT, 0, m_data.mpi_comm); @@ -223,8 +220,14 @@ size_t DataSpacesReader::CurrentStep() const { return m_CurrentStep; } void DataSpacesReader::EndStep() { - MPI_Barrier(m_data.mpi_comm); PerformGets(); + char *meta_lk; + std::string lk_name = f_Name + std::to_string(m_CurrentStep); + meta_lk = new char[lk_name.length() + 1]; + strcpy(meta_lk, lk_name.c_str()); + + MPI_Comm lock_comm = m_data.mpi_comm; + dspaces_unlock_on_read(meta_lk, &lock_comm); } void DataSpacesReader::DoClose(const int transportIndex) @@ -244,7 +247,7 @@ void DataSpacesReader::Flush(const int transportIndex) {} void DataSpacesReader::PerformGets() { - if (m_DeferredStack.size() > 0 && m_CurrentStep <= latestStep) + if (m_DeferredStack.size() > 0) { #define declare_type(T) \ for (std::string variableName : m_DeferredStack) \ diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.tcc b/source/adios2/engine/dataspaces/DataSpacesReader.tcc index f7c87697b1..00f9a998a8 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.tcc +++ b/source/adios2/engine/dataspaces/DataSpacesReader.tcc @@ -65,28 +65,37 @@ void DataSpacesReader::ReadDsData(Variable &variable, T *data, int version) i,j --> j, i = lb[1], lb[0] i --> i = lb[0] */ - - if (isOrderC) + if (variable.m_Shape.size() == 0) { - for (int i = 0; i < ndims; i++) - { - gdims_in[i] = - static_cast(variable.m_Shape[ndims - i - 1]); - lb_in[i] = static_cast(variable.m_Start[ndims - i - 1]); - ub_in[i] = - static_cast(variable.m_Start[ndims - i - 1] + - variable.m_Count[ndims - i - 1] - 1); - } + gdims_in[0] = dspaces_get_num_space_server(); + lb_in[0] = 0; + ub_in[0] = 0; + ndims = 1; } else { - - for (int i = 0; i < ndims; i++) + if (isOrderC) { - gdims_in[i] = static_cast(variable.m_Shape[i]); - lb_in[i] = static_cast(variable.m_Start[i]); - ub_in[i] = static_cast(variable.m_Start[i] + - variable.m_Count[i] - 1); + for (int i = 0; i < ndims; i++) + { + gdims_in[i] = + static_cast(variable.m_Shape[ndims - i - 1]); + lb_in[i] = static_cast(variable.m_Start[ndims - i - 1]); + ub_in[i] = + static_cast(variable.m_Start[ndims - i - 1] + + variable.m_Count[ndims - i - 1] - 1); + } + } + else + { + + for (int i = 0; i < ndims; i++) + { + gdims_in[i] = static_cast(variable.m_Shape[i]); + lb_in[i] = static_cast(variable.m_Start[i]); + ub_in[i] = static_cast(variable.m_Start[i] + + variable.m_Count[i] - 1); + } } } @@ -99,12 +108,10 @@ void DataSpacesReader::ReadDsData(Variable &variable, T *data, int version) char *cstr = new char[l_Name.length() + 1]; strcpy(cstr, l_Name.c_str()); - dspaces_lock_on_read(cstr, &m_data.mpi_comm); - dspaces_define_gdim(var_str, ndims, gdims_in); dspaces_get(var_str, version, variable.m_ElementSize, ndims, lb_in, ub_in, (void *)data); - dspaces_unlock_on_read(cstr, &m_data.mpi_comm); + delete[] cstr; delete[] var_str; } diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp index 11d97972a2..5b2acd2cf3 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp @@ -66,10 +66,16 @@ size_t DataSpacesWriter::CurrentStep() const { return m_CurrentStep; } void DataSpacesWriter::EndStep() { - int rank; - MPI_Comm_rank(m_data.mpi_comm, &rank); + std::string local_file_var; + + local_file_var = f_Name + std::to_string(m_CurrentStep); + char *meta_lk = new char[local_file_var.length() + 1]; + strcpy(meta_lk, local_file_var.c_str()); + MPI_Comm lock_comm = m_data.mpi_comm; + + dspaces_lock_on_write(meta_lk, &lock_comm); WriteVarInfo(); - MPI_Barrier(m_data.mpi_comm); + dspaces_unlock_on_write(meta_lk, &lock_comm); } void DataSpacesWriter::Flush(const int transportIndex) {} @@ -118,13 +124,8 @@ void DataSpacesWriter::WriteVarInfo() char *local_str, *buffer, *name_string; uint64_t *gdim_meta; uint64_t gdims[MAX_DS_NDIM], lb[MAX_DS_NDIM], ub[MAX_DS_NDIM]; - - local_file_var = f_Name + std::to_string(m_CurrentStep); - char *meta_lk = new char[local_file_var.length() + 1]; - strcpy(meta_lk, local_file_var.c_str()); MPI_Comm_rank(m_data.mpi_comm, &rank); - MPI_Comm lock_comm = MPI_COMM_SELF; - + if (rank == 0) { @@ -180,13 +181,11 @@ void DataSpacesWriter::WriteVarInfo() lb[0] = 0; ub[0] = buf_len - 1; gdims[0] = (ub[0] - lb[0] + 1) * dspaces_get_num_space_server(); - dspaces_lock_on_write(meta_lk, &lock_comm); dspaces_define_gdim(local_str, ndim, gdims); dspaces_put(local_str, m_CurrentStep, elemsize, ndim, lb, ub, buffer); dspaces_put_sync(); // wait on previous put to finish - dspaces_unlock_on_write(meta_lk, &lock_comm); delete[] local_str; free(dim_meta); free(elemSize_meta); diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.tcc b/source/adios2/engine/dataspaces/DataSpacesWriter.tcc index d2a185a9b2..5fd8767a4a 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.tcc +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.tcc @@ -36,7 +36,6 @@ void DataSpacesWriter::DoPutSyncCommon(Variable &variable, const T *values) unsigned int version; version = m_CurrentStep; int ndims = std::max(variable.m_Shape.size(), variable.m_Count.size()); - ndim_vector.push_back(ndims); bool isOrderC = helper::IsRowMajor(m_IO.m_HostLanguage); /* Order of dimensions: in DataSpaces: fast --> slow --> slowest For example: @@ -47,30 +46,42 @@ void DataSpacesWriter::DoPutSyncCommon(Variable &variable, const T *values) i,j --> j, i = lb[1], lb[0] i --> i = lb[0] */ - - if (isOrderC) + if (variable.m_SingleValue) { - for (int i = 0; i < ndims; i++) - { - gdims_in[i] = - static_cast(variable.m_Shape[ndims - i - 1]); - dims_vec.push_back(gdims_in[i]); - lb_in[i] = static_cast(variable.m_Start[ndims - i - 1]); - ub_in[i] = - static_cast(variable.m_Start[ndims - i - 1] + - variable.m_Count[ndims - i - 1] - 1); - } + gdims_in[0] = dspaces_get_num_space_server(); + lb_in[0] = 0; + ub_in[0] = 0; + ndims = 1; + dims_vec.push_back(0); + ndim_vector.push_back(0); } else { - - for (int i = 0; i < ndims; i++) + ndim_vector.push_back(ndims); + if (isOrderC) { - gdims_in[i] = static_cast(variable.m_Shape[i]); - dims_vec.push_back(gdims_in[i]); - lb_in[i] = static_cast(variable.m_Start[i]); - ub_in[i] = static_cast(variable.m_Start[i] + - variable.m_Count[i] - 1); + for (int i = 0; i < ndims; i++) + { + gdims_in[i] = + static_cast(variable.m_Shape[ndims - i - 1]); + dims_vec.push_back(gdims_in[i]); + lb_in[i] = static_cast(variable.m_Start[ndims - i - 1]); + ub_in[i] = + static_cast(variable.m_Start[ndims - i - 1] + + variable.m_Count[ndims - i - 1] - 1); + } + } + else + { + + for (int i = 0; i < ndims; i++) + { + gdims_in[i] = static_cast(variable.m_Shape[i]); + dims_vec.push_back(gdims_in[i]); + lb_in[i] = static_cast(variable.m_Start[i]); + ub_in[i] = static_cast(variable.m_Start[i] + + variable.m_Count[i] - 1); + } } } gdims_vector.push_back(dims_vec); @@ -98,12 +109,11 @@ void DataSpacesWriter::DoPutSyncCommon(Variable &variable, const T *values) char *cstr = new char[l_Name.length() + 1]; strcpy(cstr, l_Name.c_str()); - dspaces_lock_on_write(cstr, &m_data.mpi_comm); dspaces_define_gdim(var_str, ndims, gdims_in); dspaces_put(var_str, version, variable.m_ElementSize, ndims, lb_in, ub_in, values); dspaces_put_sync(); - dspaces_unlock_on_write(cstr, &m_data.mpi_comm); + dspaces_put_sync(); delete[] cstr; delete[] var_str; } From 8464e2208ffe139e63c3882132656b06635adff8 Mon Sep 17 00:00:00 2001 From: pradsubedi Date: Mon, 14 Oct 2019 11:22:50 -0400 Subject: [PATCH 2/3] Clang formatted --- source/adios2/engine/dataspaces/DataSpacesReader.cpp | 6 +++--- source/adios2/engine/dataspaces/DataSpacesReader.tcc | 3 ++- source/adios2/engine/dataspaces/DataSpacesWriter.cpp | 6 +++--- source/adios2/engine/dataspaces/DataSpacesWriter.tcc | 3 ++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.cpp b/source/adios2/engine/dataspaces/DataSpacesReader.cpp index 61e0c8d949..6685f2633d 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesReader.cpp @@ -83,7 +83,7 @@ StepStatus DataSpacesReader::BeginStep(StepMode mode, const float timeout_sec) MPI_Comm lock_comm = m_data.mpi_comm; dspaces_lock_on_read(meta_lk, &lock_comm); - + int nVars = 0; if (!m_ProvideLatest) { @@ -237,8 +237,8 @@ void DataSpacesReader::DoClose(const int transportIndex) !globals_adios_is_dataspaces_connected_from_both()) { // fprintf(stderr, "Disconnecting reader via finalize \n"); - MPI_Barrier(m_data.mpi_comm); - dspaces_finalize(); + // MPI_Barrier(m_data.mpi_comm); + // dspaces_finalize(); } globals_adios_set_dataspaces_disconnected_from_writer(); } diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.tcc b/source/adios2/engine/dataspaces/DataSpacesReader.tcc index 00f9a998a8..75eb20f89f 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.tcc +++ b/source/adios2/engine/dataspaces/DataSpacesReader.tcc @@ -80,7 +80,8 @@ void DataSpacesReader::ReadDsData(Variable &variable, T *data, int version) { gdims_in[i] = static_cast(variable.m_Shape[ndims - i - 1]); - lb_in[i] = static_cast(variable.m_Start[ndims - i - 1]); + lb_in[i] = + static_cast(variable.m_Start[ndims - i - 1]); ub_in[i] = static_cast(variable.m_Start[ndims - i - 1] + variable.m_Count[ndims - i - 1] - 1); diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp index 5b2acd2cf3..43ea2a2638 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp @@ -94,8 +94,8 @@ void DataSpacesWriter::DoClose(const int transportIndex) !globals_adios_is_dataspaces_connected_from_both()) { // fprintf(stderr, "Disconnecting writer via finalize \n"); - MPI_Barrier(m_data.mpi_comm); - dspaces_finalize(); + // MPI_Barrier(m_data.mpi_comm); + // dspaces_finalize(); } globals_adios_set_dataspaces_disconnected_from_writer(); } @@ -125,7 +125,7 @@ void DataSpacesWriter::WriteVarInfo() uint64_t *gdim_meta; uint64_t gdims[MAX_DS_NDIM], lb[MAX_DS_NDIM], ub[MAX_DS_NDIM]; MPI_Comm_rank(m_data.mpi_comm, &rank); - + if (rank == 0) { diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.tcc b/source/adios2/engine/dataspaces/DataSpacesWriter.tcc index 5fd8767a4a..90461a181c 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.tcc +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.tcc @@ -65,7 +65,8 @@ void DataSpacesWriter::DoPutSyncCommon(Variable &variable, const T *values) gdims_in[i] = static_cast(variable.m_Shape[ndims - i - 1]); dims_vec.push_back(gdims_in[i]); - lb_in[i] = static_cast(variable.m_Start[ndims - i - 1]); + lb_in[i] = + static_cast(variable.m_Start[ndims - i - 1]); ub_in[i] = static_cast(variable.m_Start[ndims - i - 1] + variable.m_Count[ndims - i - 1] - 1); From be4aacbb64e9f36bc5df20369b382e6ac85f55fa Mon Sep 17 00:00:00 2001 From: pradsubedi Date: Tue, 15 Oct 2019 13:47:34 -0400 Subject: [PATCH 3/3] Removed commented code blocks --- source/adios2/engine/dataspaces/DataSpacesReader.cpp | 9 +-------- source/adios2/engine/dataspaces/DataSpacesWriter.cpp | 7 ------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.cpp b/source/adios2/engine/dataspaces/DataSpacesReader.cpp index 6685f2633d..3c8a814aa1 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesReader.cpp @@ -233,14 +233,7 @@ void DataSpacesReader::EndStep() void DataSpacesReader::DoClose(const int transportIndex) { - if (globals_adios_is_dataspaces_connected_from_reader() && - !globals_adios_is_dataspaces_connected_from_both()) - { - // fprintf(stderr, "Disconnecting reader via finalize \n"); - // MPI_Barrier(m_data.mpi_comm); - // dspaces_finalize(); - } - globals_adios_set_dataspaces_disconnected_from_writer(); + globals_adios_set_dataspaces_disconnected_from_reader(); } void DataSpacesReader::Flush(const int transportIndex) {} diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp index 43ea2a2638..e645431da5 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp @@ -90,13 +90,6 @@ void DataSpacesWriter::DoClose(const int transportIndex) dspaces_lock_on_write(meta_lk, &(m_data.mpi_comm)); dspaces_unlock_on_write(meta_lk, &(m_data.mpi_comm)); - if (globals_adios_is_dataspaces_connected_from_writer() && - !globals_adios_is_dataspaces_connected_from_both()) - { - // fprintf(stderr, "Disconnecting writer via finalize \n"); - // MPI_Barrier(m_data.mpi_comm); - // dspaces_finalize(); - } globals_adios_set_dataspaces_disconnected_from_writer(); }