diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index 04155b2979..5c083c6fa8 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -349,8 +349,10 @@ void VariableBase::InitShapeType() if (m_Shape.size() == 1 && m_Shape.front() == LocalValueDim) { m_ShapeID = ShapeID::LocalValue; - m_Start.resize(1); - m_Count.resize(1); // real count = 1 + m_Start.resize(1, 0); // start[0] == 0 + m_Count.resize(1, 1); // count[0] == 1 + // Count[0] == 1 makes sure the value will be written + // into the data file (PutVariablePayload()) m_SingleValue = true; } else diff --git a/source/adios2/toolkit/format/bp4/BP4Deserializer.tcc b/source/adios2/toolkit/format/bp4/BP4Deserializer.tcc index 3947ec9761..9e17f3a9b7 100644 --- a/source/adios2/toolkit/format/bp4/BP4Deserializer.tcc +++ b/source/adios2/toolkit/format/bp4/BP4Deserializer.tcc @@ -1134,6 +1134,7 @@ std::vector::Info> BP4Deserializer::BlocksInfoCommon( std::vector::Info> blocksInfo; blocksInfo.reserve(blocksIndexOffsets.size()); + size_t n = 0; for (const size_t blockIndexOffset : blocksIndexOffsets) { size_t position = blockIndexOffset; @@ -1166,7 +1167,23 @@ std::vector::Info> BP4Deserializer::BlocksInfoCommon( blockInfo.Min = blockCharacteristics.Statistics.Min; blockInfo.Max = blockCharacteristics.Statistics.Max; } + if (blockInfo.Shape.size() == 1 && + blockInfo.Shape.front() == LocalValueDim) + { + blockInfo.Shape = Dims{blocksIndexOffsets.size()}; + blockInfo.Count = Dims{1}; + blockInfo.Start = Dims{n}; + blockInfo.Min = blockCharacteristics.Statistics.Value; + blockInfo.Max = blockCharacteristics.Statistics.Value; + } + // bp index starts at 1 + blockInfo.Step = + static_cast(blockCharacteristics.Statistics.Step - 1); + blockInfo.BlockID = n; + blocksInfo.push_back(blockInfo); + + ++n; } return blocksInfo; } diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.h b/source/adios2/toolkit/format/bp4/BP4Serializer.h index 8ea383afbf..d02bd61d06 100644 --- a/source/adios2/toolkit/format/bp4/BP4Serializer.h +++ b/source/adios2/toolkit/format/bp4/BP4Serializer.h @@ -254,11 +254,11 @@ class BP4Serializer : public BP4Base const Stats &stats, std::vector &buffer) noexcept; template - void PutVariableCharacteristics( + void PutVariableCharacteristicsInData( const core::Variable &variable, const typename core::Variable::Info &blockInfo, - const Stats &stats, std::vector &buffer, size_t &position, - const bool putDimensions = true) noexcept; + const Stats &stats, std::vector &buffer, + size_t &position) noexcept; /** * Writes from &buffer[position]: [2 diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.tcc b/source/adios2/toolkit/format/bp4/BP4Serializer.tcc index 86b688e0b7..315ae9b46b 100644 --- a/source/adios2/toolkit/format/bp4/BP4Serializer.tcc +++ b/source/adios2/toolkit/format/bp4/BP4Serializer.tcc @@ -493,8 +493,8 @@ void BP4Serializer::PutVariableMetadataInData( buffer, position); // CHARACTERISTICS - PutVariableCharacteristics(variable, blockInfo, stats, buffer, position, - false); + PutVariableCharacteristicsInData(variable, blockInfo, stats, buffer, + position); // pad metadata so that data will fall on aligned position in memory // write a padding plus block identifier VMD] @@ -662,13 +662,8 @@ void BP4Serializer::PutBoundsRecord(const bool singleValue, { if (singleValue) { - const uint8_t id = characteristic_value; - helper::CopyToBuffer(buffer, position, &id); - // special case required by bpdump - const uint16_t length = sizeof(T); - helper::CopyToBuffer(buffer, position, &length); - helper::CopyToBuffer(buffer, position, &stats.Min); - ++characteristicsCounter; + PutCharacteristicRecord(characteristic_value, characteristicsCounter, + stats.Min, buffer, position); } else { @@ -836,11 +831,10 @@ void BP4Serializer::PutVariableCharacteristics( } template -void BP4Serializer::PutVariableCharacteristics( +void BP4Serializer::PutVariableCharacteristicsInData( const core::Variable &variable, const typename core::Variable::Info &blockInfo, const Stats &stats, - std::vector &buffer, size_t &position, - const bool putDimensions) noexcept + std::vector &buffer, size_t &position) noexcept { // going back at the end const size_t characteristicsCountPosition = position; @@ -848,24 +842,10 @@ void BP4Serializer::PutVariableCharacteristics( position += 5; uint8_t characteristicsCounter = 0; - if (putDimensions) - { - // DIMENSIONS - uint8_t characteristicID = characteristic_dimensions; - helper::CopyToBuffer(buffer, position, &characteristicID); - - const uint8_t dimensions = static_cast(blockInfo.Count.size()); - helper::CopyToBuffer(buffer, position, &dimensions); // count - const uint16_t dimensionsLength = - static_cast(24 * dimensions); - helper::CopyToBuffer(buffer, position, &dimensionsLength); // length - PutDimensionsRecord(blockInfo.Count, blockInfo.Shape, blockInfo.Start, - buffer, position, true); - ++characteristicsCounter; - } - - // VALUE for SCALAR or STAT min, max for ARRAY - if (blockInfo.Data != nullptr) + // Write STAT min, max characteristics for an ARRAY + // VALUE variable data is not written into characteristics + // in the data file (only in metadata file in other function) + if (blockInfo.Data != nullptr && !variable.m_SingleValue) { PutBoundsRecord(variable.m_SingleValue, stats, characteristicsCounter, buffer, position); diff --git a/source/utils/bp4dbg/bp4dbg.py b/source/utils/bp4dbg/bp4dbg.py index 8687f32e6e..59db9a09ee 100644 --- a/source/utils/bp4dbg/bp4dbg.py +++ b/source/utils/bp4dbg/bp4dbg.py @@ -22,8 +22,8 @@ def SetupArgs(): action="store_true") parser.add_argument("--no-metadata", "-m", help="Do not print metadata md.0", action="store_true") - parser.add_argument( - "--no-data", "-d", help="Do not print data data.*", action="store_true") + parser.add_argument("--no-data", "-d", + help="Do not print data data.*", action="store_true") args = parser.parse_args() # default values diff --git a/source/utils/bp4dbg/bp4dbg_data.py b/source/utils/bp4dbg/bp4dbg_data.py index 76f990956a..84240a382c 100644 --- a/source/utils/bp4dbg/bp4dbg_data.py +++ b/source/utils/bp4dbg/bp4dbg_data.py @@ -154,7 +154,7 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize, currentPosition = f.tell() print(" Payload offset : {0}".format(currentPosition)) nBytes = np.ones(1, dtype=np.uint64) - nBytes[0] = nElements[0] * typeSize + nBytes[0] = nElements * typeSize if (currentPosition + nBytes[0] > varsStartPosition + varsTotalLength): print("ERROR: Variable data block of size would reach beyond all " "variable blocks") @@ -168,12 +168,21 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize, "calculated from var block length") print("Expected size = {0} calculated size from dimensions = {1}". format(expectedSize, nBytes[0])) - print(" Variable Data : {0} bytes".format(nBytes[0])) - # seek instead of reading for now - f.read(nBytes[0]) - # f.seek(nBytes[0], 1) - # data = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID), - # nElements) + + if nElements == 1: + # single value. read and print + value = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID), + nElements) + print(" Payload (value) : {0} ({1} bytes)".format( + value[0], nBytes[0])) + else: + # seek instead of reading for now + f.read(nBytes[0]) + # f.seek(nBytes[0], 1) + # data = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID), + # nElements) + print(" Payload (array) : {0} bytes".format(nBytes[0])) + return True # Read a variable's metadata @@ -248,8 +257,9 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength): print(" Dims Length : {0}".format( dimsLen)) - nElements = np.ones(1, dtype=np.uint64) + nElements = np.uint64(1) ldims = np.zeros(ndims, dtype=np.uint64) + isLocalValueArray = False for i in range(ndims): print(" Dim[{0}]".format(i)) # Read Local Dimensions (1 byte flag + 8 byte value) @@ -278,7 +288,11 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength): if (isDimensionVarID == b'\0'): isDimensionVarID = b'n' gdim = np.fromfile(f, dtype=np.uint64, count=1)[0] - print(" global dim : {0}".format(gdim)) + if i == 0 and ldims[i] == 0 and gdim == bp4dbg_utils.LocalValueDim: + print(" global dim : LocalValueDim ({0})".format(gdim)) + isLocalValueArray = True + else: + print(" global dim : {0}".format(gdim)) # Read Offset Dimensions (1 byte flag + 8 byte value) # Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0' @@ -310,9 +324,16 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength): print(" Tag (pad {0:2d}) : {1}".format( endTagLen - 4, tag.decode('ascii'))) - expectedVarDataSize = expectedVarBlockLength - (f.tell() - startPosition) - status = ReadVarData(f, nElements, typeID, ldims, expectedVarDataSize, - varsStartPosition, varsTotalLength) + # special case: LocalValueDim: local values turned into 1D global array + # but it seems there is no data block at all for these variables + if isLocalValueArray: + ldims[0] = 1 + nElements = np.uint64(1) + else: + expectedVarDataSize = expectedVarBlockLength - \ + (f.tell() - startPosition) + status = ReadVarData(f, nElements, typeID, ldims, expectedVarDataSize, + varsStartPosition, varsTotalLength) if not status: return False diff --git a/source/utils/bp4dbg/bp4dbg_metadata.py b/source/utils/bp4dbg/bp4dbg_metadata.py index 26f1a69f5c..3d71eaf35c 100644 --- a/source/utils/bp4dbg/bp4dbg_metadata.py +++ b/source/utils/bp4dbg/bp4dbg_metadata.py @@ -209,8 +209,10 @@ def ReadPGMD(buf, idx, pos, limit, pgStartOffset): # 2 bytes PG Length + Name length pgLength = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0] pos = pos + 2 - print(" PG length : {0} bytes".format(pgLength)) - if (pgStartPosition + pgLength > limit): + print( + " PG length : {0} bytes (+2 for length)".format( + pgLength)) + if (pgStartPosition + pgLength + 2 > limit): print("ERROR: There is not enough bytes {0} left in PG index block " "to read this single PG index ({1} bytes)").format( limit - pgStartPosition, pgLength) @@ -277,8 +279,9 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset): # 4 bytes VAR Index Length varLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0] pos = pos + 4 - print(" Var idx length : {0} bytes".format(varLength)) - if (varStartPosition + varLength > limit): + print(" Var idx length : {0} bytes (+4 for idx length)".format( + varLength)) + if (varStartPosition + varLength + 4 > limit): print("ERROR: There is not enough bytes in Var index block " "to read this single Var index") return False, pos @@ -344,8 +347,9 @@ def ReadAttrMD(buf, idx, pos, limit, attrStartOffset): # 4 bytes ATTR Index Length attrLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0] pos = pos + 4 - print(" Attr idx length : {0} bytes".format(attrLength)) - if (attrStartPosition + attrLength > limit): + print(" Attr idx length : {0} bytes (+4 for idx length)".format( + attrLength)) + if (attrStartPosition + attrLength + 4 > limit): print("ERROR: There is not enough bytes in Attr index block " "to read this single Attr index") return False, pos diff --git a/source/utils/bp4dbg/bp4dbg_utils.py b/source/utils/bp4dbg/bp4dbg_utils.py index 94cd71c01d..ba894700ef 100644 --- a/source/utils/bp4dbg/bp4dbg_utils.py +++ b/source/utils/bp4dbg/bp4dbg_utils.py @@ -1,6 +1,8 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +LocalValueDim = 18446744073709551613 + dataTypes = { -1: 'unknown', 0: 'byte',