Merge pull request ornladios#1579 from pnorbert/bp4-localvalues

- Handle LocalValueDim (local values turned into 1D global array) in …

source/adios2/core/VariableBase.cpp

@@ -349,8 +349,10 @@ void VariableBase::InitShapeType()
             if (m_Shape.size() == 1 && m_Shape.front() == LocalValueDim)
             {
                 m_ShapeID = ShapeID::LocalValue;
-                m_Start.resize(1);
-                m_Count.resize(1); // real count = 1
+                m_Start.resize(1, 0); // start[0] == 0
+                m_Count.resize(1, 1); // count[0] == 1
+                // Count[0] == 1 makes sure the value will be written
+                // into the data file (PutVariablePayload())
                 m_SingleValue = true;
             }
             else

source/adios2/toolkit/format/bp4/BP4Deserializer.tcc

@@ -1134,6 +1134,7 @@ std::vector<typename core::Variable<T>::Info> BP4Deserializer::BlocksInfoCommon(
     std::vector<typename core::Variable<T>::Info> blocksInfo;
     blocksInfo.reserve(blocksIndexOffsets.size());
 
+    size_t n = 0;
     for (const size_t blockIndexOffset : blocksIndexOffsets)
     {
         size_t position = blockIndexOffset;
@@ -1166,7 +1167,23 @@ std::vector<typename core::Variable<T>::Info> BP4Deserializer::BlocksInfoCommon(
             blockInfo.Min = blockCharacteristics.Statistics.Min;
             blockInfo.Max = blockCharacteristics.Statistics.Max;
         }
+        if (blockInfo.Shape.size() == 1 &&
+            blockInfo.Shape.front() == LocalValueDim)
+        {
+            blockInfo.Shape = Dims{blocksIndexOffsets.size()};
+            blockInfo.Count = Dims{1};
+            blockInfo.Start = Dims{n};
+            blockInfo.Min = blockCharacteristics.Statistics.Value;
+            blockInfo.Max = blockCharacteristics.Statistics.Value;
+        }
+        // bp index starts at 1
+        blockInfo.Step =
+            static_cast<size_t>(blockCharacteristics.Statistics.Step - 1);
+        blockInfo.BlockID = n;
+
         blocksInfo.push_back(blockInfo);
+
+        ++n;
     }
     return blocksInfo;
 }

source/adios2/toolkit/format/bp4/BP4Serializer.h

@@ -254,11 +254,11 @@ class BP4Serializer : public BP4Base
         const Stats<T> &stats, std::vector<char> &buffer) noexcept;
 
     template <class T>
-    void PutVariableCharacteristics(
+    void PutVariableCharacteristicsInData(
         const core::Variable<T> &variable,
         const typename core::Variable<T>::Info &blockInfo,
-        const Stats<T> &stats, std::vector<char> &buffer, size_t &position,
-        const bool putDimensions = true) noexcept;
+        const Stats<T> &stats, std::vector<char> &buffer,
+        size_t &position) noexcept;
 
     /**
      * Writes from &buffer[position]:  [2

source/adios2/toolkit/format/bp4/BP4Serializer.tcc

@@ -493,8 +493,8 @@ void BP4Serializer::PutVariableMetadataInData(
                         buffer, position);
 
     // CHARACTERISTICS
-    PutVariableCharacteristics(variable, blockInfo, stats, buffer, position,
-                               false);
+    PutVariableCharacteristicsInData(variable, blockInfo, stats, buffer,
+                                     position);
 
     // pad metadata so that data will fall on aligned position in memory
     // write a padding plus block identifier VMD]
@@ -662,13 +662,8 @@ void BP4Serializer::PutBoundsRecord(const bool singleValue,
 {
     if (singleValue)
     {
-        const uint8_t id = characteristic_value;
-        helper::CopyToBuffer(buffer, position, &id);
-        // special case required by bpdump
-        const uint16_t length = sizeof(T);
-        helper::CopyToBuffer(buffer, position, &length);
-        helper::CopyToBuffer(buffer, position, &stats.Min);
-        ++characteristicsCounter;
+        PutCharacteristicRecord(characteristic_value, characteristicsCounter,
+                                stats.Min, buffer, position);
     }
     else
     {
@@ -836,36 +831,21 @@ void BP4Serializer::PutVariableCharacteristics(
 }
 
 template <class T>
-void BP4Serializer::PutVariableCharacteristics(
+void BP4Serializer::PutVariableCharacteristicsInData(
     const core::Variable<T> &variable,
     const typename core::Variable<T>::Info &blockInfo, const Stats<T> &stats,
-    std::vector<char> &buffer, size_t &position,
-    const bool putDimensions) noexcept
+    std::vector<char> &buffer, size_t &position) noexcept
 {
     // going back at the end
     const size_t characteristicsCountPosition = position;
     // skip characteristics count(1) + length (4)
     position += 5;
     uint8_t characteristicsCounter = 0;
 
-    if (putDimensions)
-    {
-        // DIMENSIONS
-        uint8_t characteristicID = characteristic_dimensions;
-        helper::CopyToBuffer(buffer, position, &characteristicID);
-
-        const uint8_t dimensions = static_cast<uint8_t>(blockInfo.Count.size());
-        helper::CopyToBuffer(buffer, position, &dimensions); // count
-        const uint16_t dimensionsLength =
-            static_cast<uint16_t>(24 * dimensions);
-        helper::CopyToBuffer(buffer, position, &dimensionsLength); // length
-        PutDimensionsRecord(blockInfo.Count, blockInfo.Shape, blockInfo.Start,
-                            buffer, position, true);
-        ++characteristicsCounter;
-    }
-
-    // VALUE for SCALAR or STAT min, max for ARRAY
-    if (blockInfo.Data != nullptr)
+    // Write STAT min, max characteristics for an ARRAY
+    // VALUE variable data is not written into characteristics
+    // in the data file (only in metadata file in other function)
+    if (blockInfo.Data != nullptr && !variable.m_SingleValue)
     {
         PutBoundsRecord(variable.m_SingleValue, stats, characteristicsCounter,
                         buffer, position);

source/utils/bp4dbg/bp4dbg.py

@@ -22,8 +22,8 @@ def SetupArgs():
                         action="store_true")
     parser.add_argument("--no-metadata", "-m",
                         help="Do not print metadata md.0", action="store_true")
-    parser.add_argument(
-        "--no-data", "-d", help="Do not print data data.*", action="store_true")
+    parser.add_argument("--no-data", "-d",
+                        help="Do not print data data.*", action="store_true")
     args = parser.parse_args()
 
     # default values

source/utils/bp4dbg/bp4dbg_data.py

@@ -154,7 +154,7 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize,
     currentPosition = f.tell()
     print("      Payload offset  : {0}".format(currentPosition))
     nBytes = np.ones(1, dtype=np.uint64)
-    nBytes[0] = nElements[0] * typeSize
+    nBytes[0] = nElements * typeSize
     if (currentPosition + nBytes[0] > varsStartPosition + varsTotalLength):
         print("ERROR: Variable data block of size would reach beyond all "
               "variable blocks")
@@ -168,12 +168,21 @@ def ReadVarData(f, nElements, typeID, ldims, expectedSize,
               "calculated from var block length")
         print("Expected size = {0}  calculated size from dimensions = {1}".
               format(expectedSize, nBytes[0]))
-    print("      Variable Data   : {0} bytes".format(nBytes[0]))
-    # seek instead of reading for now
-    f.read(nBytes[0])
-    # f.seek(nBytes[0], 1)
-    # data = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID),
-    #                            nElements)
+
+    if nElements == 1:
+        # single value. read and print
+        value = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID),
+                                     nElements)
+        print("      Payload (value) : {0} ({1} bytes)".format(
+            value[0], nBytes[0]))
+    else:
+        # seek instead of reading for now
+        f.read(nBytes[0])
+        # f.seek(nBytes[0], 1)
+        # data = readDataToNumpyArray(f, bp4dbg_utils.GetTypeName(typeID),
+        #                            nElements)
+        print("      Payload (array) : {0} bytes".format(nBytes[0]))
+
     return True
 
 # Read a variable's metadata
@@ -248,8 +257,9 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
     print("      Dims Length     : {0}".format(
         dimsLen))
 
-    nElements = np.ones(1, dtype=np.uint64)
+    nElements = np.uint64(1)
     ldims = np.zeros(ndims, dtype=np.uint64)
+    isLocalValueArray = False
     for i in range(ndims):
         print("      Dim[{0}]".format(i))
         # Read Local Dimensions (1 byte flag + 8 byte value)
@@ -278,7 +288,11 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
         if (isDimensionVarID == b'\0'):
             isDimensionVarID = b'n'
         gdim = np.fromfile(f, dtype=np.uint64, count=1)[0]
-        print("           global dim : {0}".format(gdim))
+        if i == 0 and ldims[i] == 0 and gdim == bp4dbg_utils.LocalValueDim:
+            print("           global dim : LocalValueDim ({0})".format(gdim))
+            isLocalValueArray = True
+        else:
+            print("           global dim : {0}".format(gdim))
 
         # Read Offset Dimensions (1 byte flag + 8 byte value)
         # Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0'
@@ -310,9 +324,16 @@ def ReadVMD(f, varidx, varsStartPosition, varsTotalLength):
     print("      Tag (pad {0:2d})    : {1}".format(
         endTagLen - 4, tag.decode('ascii')))
 
-    expectedVarDataSize = expectedVarBlockLength - (f.tell() - startPosition)
-    status = ReadVarData(f, nElements, typeID, ldims, expectedVarDataSize,
-                         varsStartPosition, varsTotalLength)
+    # special case: LocalValueDim: local values turned into 1D global array
+    # but it seems there is no data block at all for these variables
+    if isLocalValueArray:
+        ldims[0] = 1
+        nElements = np.uint64(1)
+    else:
+        expectedVarDataSize = expectedVarBlockLength - \
+            (f.tell() - startPosition)
+        status = ReadVarData(f, nElements, typeID, ldims, expectedVarDataSize,
+                             varsStartPosition, varsTotalLength)
     if not status:
         return False
 

source/utils/bp4dbg/bp4dbg_metadata.py

@@ -209,8 +209,10 @@ def ReadPGMD(buf, idx, pos, limit, pgStartOffset):
     # 2 bytes PG Length + Name length
     pgLength = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0]
     pos = pos + 2
-    print("        PG length       : {0} bytes".format(pgLength))
-    if (pgStartPosition + pgLength > limit):
+    print(
+        "        PG length       : {0} bytes (+2 for length)".format(
+            pgLength))
+    if (pgStartPosition + pgLength + 2 > limit):
         print("ERROR: There is not enough bytes {0} left in PG index block "
               "to read this single PG index ({1} bytes)").format(
                   limit - pgStartPosition, pgLength)
@@ -277,8 +279,9 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset):
     # 4 bytes VAR Index Length
     varLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0]
     pos = pos + 4
-    print("        Var idx length  : {0} bytes".format(varLength))
-    if (varStartPosition + varLength > limit):
+    print("        Var idx length  : {0} bytes (+4 for idx length)".format(
+        varLength))
+    if (varStartPosition + varLength + 4 > limit):
         print("ERROR: There is not enough bytes in Var index block "
               "to read this single Var index")
         return False, pos
@@ -344,8 +347,9 @@ def ReadAttrMD(buf, idx, pos, limit, attrStartOffset):
     # 4 bytes ATTR Index Length
     attrLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0]
     pos = pos + 4
-    print("        Attr idx length : {0} bytes".format(attrLength))
-    if (attrStartPosition + attrLength > limit):
+    print("        Attr idx length : {0} bytes (+4 for idx length)".format(
+        attrLength))
+    if (attrStartPosition + attrLength + 4 > limit):
         print("ERROR: There is not enough bytes in Attr index block "
               "to read this single Attr index")
         return False, pos

source/utils/bp4dbg/bp4dbg_utils.py

@@ -1,6 +1,8 @@
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
 
+LocalValueDim = 18446744073709551613
+
 dataTypes = {
     -1: 'unknown',
     0: 'byte',