diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index 9ba709ef66..0b0eaea8bd 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -1756,6 +1756,11 @@ void BP5Writer::PutCommon(VariableBase &variable, const void *values, bool sync) { Count = variable.m_Count.data(); } + else if (variable.m_ShapeID == ShapeID::JoinedArray) + { + Count = variable.m_Count.data(); + Shape = variable.m_Shape.data(); + } size_t ObjSize; if (variable.m_Type == DataType::Struct) diff --git a/source/adios2/engine/bp5/BP5Writer.tcc b/source/adios2/engine/bp5/BP5Writer.tcc index 3e059a1a5a..0a4bdec02d 100644 --- a/source/adios2/engine/bp5/BP5Writer.tcc +++ b/source/adios2/engine/bp5/BP5Writer.tcc @@ -41,6 +41,12 @@ void BP5Writer::PutCommonSpan(Variable &variable, Start = variable.m_Start.data(); Count = variable.m_Count.data(); } + else if (variable.m_ShapeID == ShapeID::JoinedArray) + { + Shape = variable.m_Shape.data(); + DimCount = variable.m_Count.size(); + Count = variable.m_Count.data(); + } else if (variable.m_ShapeID == ShapeID::LocalArray) { DimCount = variable.m_Count.size(); diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index d7e4d0d806..8519c50208 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -592,9 +592,16 @@ void BP5Deserializer::SetupForStep(size_t Step, size_t WriterCount) { m_Engine->m_IO.RemoveVariable(RecPair.second->VarName); RecPair.second->Variable = NULL; + if (RecPair.second->OrigShapeID == ShapeID::JoinedArray) + { + auto VarRec = RecPair.second; + VarRec->JoinedDimen = SIZE_MAX; + VarRec->LastJoinedOffset = NULL; + VarRec->LastJoinedShape = NULL; + } } - m_CurrentWriterCohortSize = WriterCount; } + m_CurrentWriterCohortSize = WriterCount; } size_t BP5Deserializer::WriterCohortSize(size_t Step) const @@ -693,6 +700,15 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, MetadataBaseArray[Step] = m_MetadataBaseAddrs; m_FreeableMBA = nullptr; } + + JoinedDimArray.resize(Step + 1); + if (JoinedDimArray[Step] == nullptr) + { + m_JoinedDimenOffsetArrays = new std::vector(); + m_JoinedDimenOffsetArrays->resize(writerCohortSize); + JoinedDimArray[Step] = m_JoinedDimenOffsetArrays; + m_FreeableMBA = nullptr; + } } else { @@ -705,9 +721,56 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, { m_MetadataBaseAddrs->resize(writerCohortSize); } + + if (!m_JoinedDimenOffsetArrays) + { + m_JoinedDimenOffsetArrays = new std::vector(); + m_FreeableMBA = m_JoinedDimenOffsetArrays; + } + if (writerCohortSize > m_JoinedDimenOffsetArrays->size()) + { + m_JoinedDimenOffsetArrays->resize(writerCohortSize); + } } (*m_MetadataBaseAddrs)[WriterRank] = BaseData; + size_t JoinedDimenTotal = 0; + for (int i = 0; i < Control->ControlCount; i++) + { + size_t FieldOffset = ControlFields[i].FieldOffset; + BP5VarRec *VarRec = ControlFields[i].VarRec; + void *field_data = (char *)BaseData + FieldOffset; + if (!BP5BitfieldTest((BP5MetadataInfoStruct *)BaseData, i)) + { + continue; + } + if (ControlFields[i].OrigShapeID == ShapeID::JoinedArray) + { + MetaArrayRec *meta_base = (MetaArrayRec *)field_data; + JoinedDimenTotal += meta_base->DBCount; + if (VarRec->JoinedDimen == SIZE_MAX) + { + for (size_t i = 0; i < meta_base->Dims; i++) + { + if (meta_base->Shape[i] == JoinedDim) + { + VarRec->JoinedDimen = i; + } + } + } + } + } + + // Allocate memory to hold new offset values for Joined Arrays + size_t CurJoinedDimenOffset = 0; + size_t *JoinedDimenOffsetArray = NULL; + if (JoinedDimenTotal) + JoinedDimenOffsetArray = (size_t *)malloc( + JoinedDimenTotal * writerCohortSize * sizeof(size_t)); + + // store this away so it can be deallocated later + (*m_JoinedDimenOffsetArrays)[WriterRank] = JoinedDimenOffsetArray; + for (int i = 0; i < Control->ControlCount; i++) { size_t FieldOffset = ControlFields[i].FieldOffset; @@ -735,7 +798,8 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, } } if ((ControlFields[i].OrigShapeID == ShapeID::GlobalArray) || - (ControlFields[i].OrigShapeID == ShapeID::LocalArray)) + (ControlFields[i].OrigShapeID == ShapeID::LocalArray) || + (ControlFields[i].OrigShapeID == ShapeID::JoinedArray)) { MetaArrayRec *meta_base = (MetaArrayRec *)field_data; size_t BlockCount = @@ -747,11 +811,15 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, * switcheroo */ ReverseDimensions(meta_base->Count, meta_base->Dims, BlockCount); - if (ControlFields[i].OrigShapeID == ShapeID::GlobalArray) + if ((ControlFields[i].OrigShapeID == ShapeID::GlobalArray) || + (ControlFields[i].OrigShapeID == ShapeID::JoinedArray)) { ReverseDimensions(meta_base->Shape, meta_base->Dims, 1); - ReverseDimensions(meta_base->Offsets, meta_base->Dims, - BlockCount); + if (ControlFields[i].OrigShapeID == ShapeID::GlobalArray) + { + ReverseDimensions(meta_base->Offsets, meta_base->Dims, + BlockCount); + } } } if ((WriterRank == 0) || (VarRec->GlobalDims == NULL)) @@ -759,6 +827,52 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, // use the shape from rank 0 (or first non-NULL) VarRec->GlobalDims = meta_base->Shape; } + if (ControlFields[i].OrigShapeID == ShapeID::JoinedArray) + { + // setup Offsets + meta_base->Offsets = + &JoinedDimenOffsetArray[CurJoinedDimenOffset]; + CurJoinedDimenOffset += meta_base->DBCount; + + for (size_t b = 0; b < BlockCount; b++) + { + size_t PreviousJoinedOffset = 0; + if (VarRec->LastJoinedShape != NULL) + { + // Offset of this new block is the prior total size, + // stored in Shape + PreviousJoinedOffset = + VarRec->LastJoinedShape[VarRec->JoinedDimen]; + } + else + { + // We're going to track the accumulated Joined Shape in + // whatever Shape metadata entry we selected for + // GlobalDims (might be rank 0, might be other) + VarRec->LastJoinedShape = VarRec->GlobalDims; + // overwrite the JoinedDimen value in that entry + VarRec->LastJoinedShape[VarRec->JoinedDimen] = 0; + } + VarRec->LastJoinedShape[VarRec->JoinedDimen] += + meta_base->Count[(b * meta_base->Dims) + + VarRec->JoinedDimen]; + for (size_t i = 0; i < meta_base->Dims; i++) + { + if (i == VarRec->JoinedDimen) + { + meta_base->Offsets[(b * meta_base->Dims) + i] = + PreviousJoinedOffset; + } + else + { + meta_base->Offsets[(b * meta_base->Dims) + i] = 0; + } + } + VarRec->LastJoinedOffset = + &meta_base->Offsets[(b * meta_base->Dims)]; + } + } + if (!VarRec->Variable) { VarRec->Variable = ArrayVarSetup( @@ -834,6 +948,27 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, VarRec->LastTSAdded = Step; } } + if (WriterRank == (m_CurrentWriterCohortSize - 1)) + { + // do step finalization procedures + if (!m_RandomAccessMode) + { + for (auto RecPair : VarByKey) + { + if (RecPair.second->Variable != NULL) + if (RecPair.second->OrigShapeID == ShapeID::JoinedArray) + { + auto VarRec = RecPair.second; + VariableBase *Var = + static_cast(VarRec->Variable); + for (size_t i = 0; i < VarRec->DimCount; i++) + { + Var->m_Shape[i] = VarRec->GlobalDims[i]; + } + } + } + } + } } void BP5Deserializer::InstallAttributeData(void *AttributeBlock, @@ -1249,7 +1384,8 @@ bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable, } MemorySpace MemSpace = variable.GetMemorySpace(DestData); if ((variable.m_SelectionType == adios2::SelectionType::BoundingBox) && - (variable.m_ShapeID == ShapeID::GlobalArray)) + ((variable.m_ShapeID == ShapeID::GlobalArray) || + (variable.m_ShapeID == ShapeID::JoinedArray))) { BP5ArrayRequest Req; Req.VarRec = VarRec; @@ -2112,7 +2248,8 @@ bool BP5Deserializer::VarShape(const VariableBase &Var, const size_t RelStep, Dims &Shape) const { BP5VarRec *VarRec = LookupVarByKey((void *)&Var); - if (VarRec->OrigShapeID != ShapeID::GlobalArray) + if (!((VarRec->OrigShapeID == ShapeID::GlobalArray) || + (VarRec->OrigShapeID == ShapeID::JoinedArray))) { return false; } @@ -2151,6 +2288,7 @@ bool BP5Deserializer::VariableMinMax(const VariableBase &Var, const size_t Step, { BP5VarRec *VarRec = LookupVarByKey((void *)&Var); if ((VarRec->OrigShapeID == ShapeID::LocalArray) || + (VarRec->OrigShapeID == ShapeID::JoinedArray) || (VarRec->OrigShapeID == ShapeID::GlobalArray)) { if (VarRec->MinMaxOffset == SIZE_MAX) @@ -2174,6 +2312,7 @@ bool BP5Deserializer::VariableMinMax(const VariableBase &Var, const size_t Step, for (size_t RelStep = StartStep; RelStep < StopStep; RelStep++) { if ((VarRec->OrigShapeID == ShapeID::LocalArray) || + (VarRec->OrigShapeID == ShapeID::JoinedArray) || (VarRec->OrigShapeID == ShapeID::GlobalArray)) { for (size_t WriterRank = 0; WriterRank < writerCohortSize; diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.h b/source/adios2/toolkit/format/bp5/BP5Deserializer.h index f662695c3c..b7f6e2e082 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.h +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.h @@ -102,6 +102,9 @@ class BP5Deserializer : virtual public BP5Base void *Variable = NULL; char *VarName = NULL; size_t DimCount = 0; + size_t JoinedDimen = SIZE_MAX; + size_t *LastJoinedOffset = NULL; + size_t *LastJoinedShape = NULL; ShapeID OrigShapeID; core::StructDefinition *Def = nullptr; core::StructDefinition *ReaderDef = nullptr; @@ -170,12 +173,17 @@ class BP5Deserializer : virtual public BP5Base nullptr; // may be a pointer into MetadataBaseArray or m_FreeableMBA std::vector *m_FreeableMBA = nullptr; + std::vector *m_JoinedDimenOffsetArrays = nullptr; + // for random access mode, for each timestep, for each writerrank, what // metameta info applies to the metadata std::vector> m_ControlArray; // for random access mode, for each timestep, for each writerrank, base // address of the metadata std::vector *> MetadataBaseArray; + // for random access mode, for each timestep, for each writerrank, base + // address of the joined dim arrays + std::vector *> JoinedDimArray; ControlInfo *ControlBlocks = nullptr; ControlInfo *GetPriorControl(FMFormat Format); diff --git a/testing/adios2/engine/staging-common/CMakeLists.txt b/testing/adios2/engine/staging-common/CMakeLists.txt index 07bbb71f4d..fd4785b719 100644 --- a/testing/adios2/engine/staging-common/CMakeLists.txt +++ b/testing/adios2/engine/staging-common/CMakeLists.txt @@ -235,8 +235,6 @@ endif() # BP5 tests if(ADIOS2_HAVE_BP5) set (BP5_TESTS ${ALL_SIMPLE_TESTS} ${SPECIAL_TESTS}) - # BP5 doesn't do Joined - list (FILTER BP5_TESTS EXCLUDE REGEX "Joined.*") # Delayed reader not worth testing on file engines list (FILTER BP5_TESTS EXCLUDE REGEX "DelayedReader") # Discard not a feature of BP5 diff --git a/testing/adios2/engine/staging-common/TestReadJoined.cpp b/testing/adios2/engine/staging-common/TestReadJoined.cpp index 91fc522a78..94a8fc45a0 100644 --- a/testing/adios2/engine/staging-common/TestReadJoined.cpp +++ b/testing/adios2/engine/staging-common/TestReadJoined.cpp @@ -90,6 +90,7 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8) int Nrows; reader.Get(rows_var, Nrows); + std::cout << "Reader expecting " << Nrows << std::endl; EXPECT_EQ(var.Shape()[0], Nrows); EXPECT_EQ(var.Shape()[1], Ncols); diff --git a/testing/adios2/engine/staging-common/TestWriteJoined.cpp b/testing/adios2/engine/staging-common/TestWriteJoined.cpp index 1799f7958b..9cda667a85 100644 --- a/testing/adios2/engine/staging-common/TestWriteJoined.cpp +++ b/testing/adios2/engine/staging-common/TestWriteJoined.cpp @@ -111,6 +111,8 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite) } if (mpiRank == 0) { + std::cout << "Writer Generating " << nTotalRows[step] << " in total" + << std::endl; writer.Put(rows_var, nTotalRows[step]); } for (int block = 0; block < nblocks; ++block)