Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BP5 subformats #3260

Merged
merged 6 commits into from
Jun 29, 2022
Merged

BP5 subformats #3260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a20d928
BP5 subformats
eisenhauer Jun 27, 2022
e01ba94
kill assert, fix typename
eisenhauer Jun 27, 2022
1f52680
Remove unused
eisenhauer Jun 27, 2022
46e5bcb
field overlap
eisenhauer Jun 27, 2022
4b8415e
Throw an exception when reading old style metadata
eisenhauer Jun 27, 2022
37ad960
Backwards compatible with prior BP5 metadata format
eisenhauer Jun 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 101 additions & 0 deletions source/adios2/toolkit/format/bp5/BP5Base.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,5 +49,106 @@ int BP5Base::BP5BitfieldTest(struct BP5MetadataInfoStruct *MBase, int Bit) const
return ((MBase->BitField[Element] & ((size_t)1 << ElementBit)) ==
((size_t)1 << ElementBit));
}
#define BASE_FIELD_ENTRIES \
{"Dims", "integer", sizeof(size_t), \
FMOffset(BP5Base::MetaArrayRec *, Dims)}, \
{"BlockCount", "integer", sizeof(size_t), \
FMOffset(BP5Base::MetaArrayRec *, BlockCount)}, \
{"DBCount", "integer", sizeof(size_t), \
FMOffset(BP5Base::MetaArrayRec *, DBCount)}, \
{"Shape", "integer[Dims]", sizeof(size_t), \
FMOffset(BP5Base::MetaArrayRec *, Shape)}, \
{"Count", "integer[DBCount]", sizeof(size_t), \
FMOffset(BP5Base::MetaArrayRec *, Count)}, \
{"Offset", "integer[DBCount]", sizeof(size_t), \
FMOffset(BP5Base::MetaArrayRec *, Offsets)}, \
{"DataBlockLocation", "integer[BlockCount]", sizeof(size_t), \
FMOffset(BP5Base::MetaArrayRec *, DataBlockLocation)},

static FMField MetaArrayRecList[] = {BASE_FIELD_ENTRIES{NULL, NULL, 0, 0}};

static FMField MetaArrayRecOperatorList[] = {
BASE_FIELD_ENTRIES{
"DataBlockSize", "integer[BlockCount]", sizeof(size_t),
FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)},
{NULL, NULL, 0, 0}};

static FMField MetaArrayRecMM1List[] = {
BASE_FIELD_ENTRIES{"MinMax", "char[2][BlockCount]", 1,
FMOffset(BP5Base::MetaArrayRecMM *, MinMax)},
{NULL, NULL, 0, 0}};

static FMField MetaArrayRecOperatorMM1List[] = {
BASE_FIELD_ENTRIES{
"DataBlockSize", "integer[BlockCount]", sizeof(size_t),
FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)},
{"MinMax", "char[2][BlockCount]", 1,
FMOffset(BP5Base::MetaArrayRecOperatorMM *, MinMax)},
{NULL, NULL, 0, 0}};
static FMField MetaArrayRecMM2List[] = {
BASE_FIELD_ENTRIES{"MinMax", "char[4][BlockCount]", 1,
FMOffset(BP5Base::MetaArrayRecMM *, MinMax)},
{NULL, NULL, 0, 0}};

static FMField MetaArrayRecOperatorMM2List[] = {
BASE_FIELD_ENTRIES{
"DataBlockSize", "integer[BlockCount]", sizeof(size_t),
FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)},
{"MinMax", "char[4][BlockCount]", 1,
FMOffset(BP5Base::MetaArrayRecOperatorMM *, MinMax)},
{NULL, NULL, 0, 0}};
static FMField MetaArrayRecMM4List[] = {
BASE_FIELD_ENTRIES{"MinMax", "char[8][BlockCount]", 1,
FMOffset(BP5Base::MetaArrayRecMM *, MinMax)},
{NULL, NULL, 0, 0}};

static FMField MetaArrayRecOperatorMM4List[] = {
BASE_FIELD_ENTRIES{
"DataBlockSize", "integer[BlockCount]", sizeof(size_t),
FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)},
{"MinMax", "char[8][BlockCount]", 1,
FMOffset(BP5Base::MetaArrayRecOperatorMM *, MinMax)},
{NULL, NULL, 0, 0}};
static FMField MetaArrayRecMM8List[] = {
BASE_FIELD_ENTRIES{"MinMax", "char[16][BlockCount]", 1,
FMOffset(BP5Base::MetaArrayRecMM *, MinMax)},
{NULL, NULL, 0, 0}};

static FMField MetaArrayRecOperatorMM8List[] = {
BASE_FIELD_ENTRIES{
"DataBlockSize", "integer[BlockCount]", sizeof(size_t),
FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)},
{"MinMax", "char[16][BlockCount]", 1,
FMOffset(BP5Base::MetaArrayRecOperatorMM *, MinMax)},
{NULL, NULL, 0, 0}};
static FMField MetaArrayRecMM16List[] = {
BASE_FIELD_ENTRIES{"MinMax", "char[32][BlockCount]", 1,
FMOffset(BP5Base::MetaArrayRecMM *, MinMax)},
{NULL, NULL, 0, 0}};

static FMField MetaArrayRecOperatorMM16List[] = {
BASE_FIELD_ENTRIES{
"DataBlockSize", "integer[BlockCount]", sizeof(size_t),
FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)},
{"MinMax", "char[32][BlockCount]", 1,
FMOffset(BP5Base::MetaArrayRecOperatorMM *, MinMax)},
{NULL, NULL, 0, 0}};
#undef BASE_FIELD_ENTRIES

BP5Base::BP5Base()
{
MetaArrayRecListPtr = &MetaArrayRecList[0];
MetaArrayRecOperatorListPtr = &MetaArrayRecOperatorList[0];
MetaArrayRecMM1ListPtr = &MetaArrayRecMM1List[0];
MetaArrayRecOperatorMM1ListPtr = &MetaArrayRecOperatorMM1List[0];
MetaArrayRecMM2ListPtr = &MetaArrayRecMM2List[0];
MetaArrayRecOperatorMM2ListPtr = &MetaArrayRecOperatorMM2List[0];
MetaArrayRecMM4ListPtr = &MetaArrayRecMM4List[0];
MetaArrayRecOperatorMM4ListPtr = &MetaArrayRecOperatorMM4List[0];
MetaArrayRecMM8ListPtr = &MetaArrayRecMM8List[0];
MetaArrayRecOperatorMM8ListPtr = &MetaArrayRecOperatorMM8List[0];
MetaArrayRecMM16ListPtr = &MetaArrayRecMM16List[0];
MetaArrayRecOperatorMM16ListPtr = &MetaArrayRecOperatorMM16List[0];
}
}
}
57 changes: 42 additions & 15 deletions source/adios2/toolkit/format/bp5/BP5Base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ namespace format
class BP5Base
{
public:
BP5Base();

struct MetaMetaInfoBlock
{
char *MetaMetaInfo;
Expand All @@ -35,29 +37,42 @@ class BP5Base
size_t MetaMetaIDLen;
};

#define BASE_FIELDS \
size_t Dims; /* How many dimensions does this array have */ \
size_t BlockCount; /* How many blocks are written */ \
size_t DBCount; /* Dimens * BlockCount */ \
size_t *Shape; /* Global dimensionality [Dims] NULL for local */ \
size_t *Count; /* Per-block Counts [DBCount] */ \
size_t *Offsets; /* Per-block Offsets [DBCount] NULL for local \
*/ \
size_t *DataBlockLocation; /* Per-block Offset in PG [BlockCount] */

typedef struct _MetaArrayRec
{
size_t Dims; // How many dimensions does this array have
size_t BlockCount; // How many blocks are written
size_t DBCount; // Dimens * BlockCount
size_t *Shape; // Global dimensionality [Dims] NULL for local
size_t *Count; // Per-block Counts [DBCount]
size_t *Offsets; // Per-block Offsets [DBCount] NULL for local
size_t *DataLocation; // Per-block Offsets [BlockCount]
BASE_FIELDS
} MetaArrayRec;

typedef struct _MetaArrayRecOperator
{
size_t Dims; // How many dimensions does this array have
size_t BlockCount; // How many blocks are written
size_t DBCount; // Dimens * BlockCount
size_t *Shape; // Global dimensionality [Dims] NULL for local
size_t *Count; // Per-block Counts [DBCount]
size_t *Offsets; // Per-block Offsets [DBCount] NULL for local
size_t *DataLocation; // Per-block Offsets [BlockCount]
size_t *DataLengths; // Per-block Lengths [BlockCount]
BASE_FIELDS
size_t *DataBlockSize; // Per-block Lengths [BlockCount]
} MetaArrayRecOperator;

typedef struct _MetaArrayRecMM
{
BASE_FIELDS
char *MinMax; // char[TYPESIZE][BlockCount] varies by type
} MetaArrayRecMM;

typedef struct _MetaArrayRecOperatorMM
{
BASE_FIELDS
size_t *DataBlockSize; // Per-block Lengths [BlockCount]
char *MinMax; // char[TYPESIZE][BlockCount] varies by type
} MetaArrayRecOperatorMM;

#undef BASE_FIELDS

struct BP5MetadataInfoStruct
{
size_t BitFieldCount;
Expand All @@ -67,6 +82,18 @@ class BP5Base

void BP5BitfieldSet(struct BP5MetadataInfoStruct *MBase, int Bit) const;
int BP5BitfieldTest(struct BP5MetadataInfoStruct *MBase, int Bit) const;
FMField *MetaArrayRecListPtr;
FMField *MetaArrayRecOperatorListPtr;
FMField *MetaArrayRecMM1ListPtr;
FMField *MetaArrayRecOperatorMM1ListPtr;
FMField *MetaArrayRecMM2ListPtr;
FMField *MetaArrayRecOperatorMM2ListPtr;
FMField *MetaArrayRecMM4ListPtr;
FMField *MetaArrayRecOperatorMM4ListPtr;
FMField *MetaArrayRecMM8ListPtr;
FMField *MetaArrayRecOperatorMM8ListPtr;
FMField *MetaArrayRecMM16ListPtr;
FMField *MetaArrayRecOperatorMM16ListPtr;
};
} // end namespace format
} // end namespace adios2
Expand Down
112 changes: 75 additions & 37 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,32 +171,55 @@ void BP5Deserializer::BreakdownVarName(const char *Name, char **base_name_p,
*base_name_p = strdup(NameStart);
}

void BP5Deserializer::BreakdownArrayName(const char *Name, char **base_name_p,
DataType *type_p, int *element_size_p,
char **Operator, bool *MinMax)
void BP5Deserializer::BreakdownFieldType(const char *FieldType, bool &Operator,
bool &MinMax)
{
if (FieldType[0] != 'M')
{
throw std::runtime_error(
"BP5 unable to parse metadata, likely old version");
}

// should start with "MetaArray"
FieldType += strlen("MetaArray");
if (FieldType == 0)
return;
if (FieldType[0] == 'O')
{
Operator = true;
FieldType += strlen("Op");
}
if (FieldType[0] == 'M')
{
MinMax = true;
}
}

void BP5Deserializer::BreakdownV1ArrayName(const char *Name, char **base_name_p,
DataType *type_p,
int *element_size_p, bool &Operator,
bool &MinMax)
{
int Type;
int ElementSize;

const char *NameStart = strchr(strchr(Name + 4, '_') + 1, '_') + 1;
// + 3 to skip BP5_ or bp5_ prefix
sscanf(Name + 4, "%d_%d", &ElementSize, &Type);
const char *Plus = index(Name, '+');
*Operator = NULL;
*MinMax = false;
MinMax = false;
while (Plus && (*Plus == '+'))
{
int Len;
if (sscanf(Plus, "+%dO", &Len) == 1)
{ // Operator Spec
*Operator = (char *)malloc(Len + 1);
Operator = true;
const char *OpStart = index(Plus, 'O') + 1;
memcpy(*Operator, index(Plus, 'O') + 1, Len);
(*Operator)[Len] = 0;
Plus = OpStart + Len;
}
else if (strncmp(Plus, "+MM", 3) == 0)
{
*MinMax = true;
MinMax = true;
Plus += 3;
}
}
Expand All @@ -206,6 +229,19 @@ void BP5Deserializer::BreakdownArrayName(const char *Name, char **base_name_p,
*(rindex(*base_name_p, '_')) = 0;
}

void BP5Deserializer::BreakdownArrayName(const char *Name, char **base_name_p,
DataType *type_p, int *element_size_p)
{
int Type;
int ElementSize;
const char *NameStart = strchr(strchr(Name + 4, '_') + 1, '_') + 1;
// + 3 to skip BP5_ or bp5_ prefix
sscanf(Name + 4, "%d_%d", &ElementSize, &Type);
*element_size_p = ElementSize;
*type_p = (DataType)Type;
*base_name_p = strdup(NameStart);
}

BP5Deserializer::BP5VarRec *BP5Deserializer::LookupVarByKey(void *Key) const
{
auto ret = VarByKey.find(Key);
Expand Down Expand Up @@ -283,18 +319,31 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format)
char *ArrayName;
DataType Type;
int ElementSize;
char *Operator = NULL;
bool Operator = false;
bool MinMax = false;
BreakdownArrayName(FieldList[i + 4].field_name, &ArrayName, &Type,
&ElementSize, &Operator, &MinMax);
bool V1_fields = true;
if (FieldList[i].field_type[0] == 'M')
V1_fields = false;
if (V1_fields)
{
BreakdownV1ArrayName(FieldList[i + 4].field_name, &ArrayName,
&Type, &ElementSize, Operator, MinMax);
}
else
{
BreakdownFieldType(FieldList[i].field_type, Operator, MinMax);
BreakdownArrayName(FieldList[i].field_name, &ArrayName, &Type,
&ElementSize);
}
VarRec = LookupVarByName(ArrayName);
if (!VarRec)
{
VarRec = CreateVarRec(ArrayName);
VarRec->Type = Type;
VarRec->ElementSize = ElementSize;
VarRec->OrigShapeID = C->OrigShapeID;
VarRec->Operator = Operator;
if (Operator)
VarRec->Operator = strdup("SomeOperator");
C->ElementSize = ElementSize;
}
C->VarRec = VarRec;
Expand All @@ -309,7 +358,14 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format)
VarRec->MinMaxOffset = MetaRecFields * sizeof(void *);
MetaRecFields++;
}
i += MetaRecFields;
if (V1_fields)
{
i += MetaRecFields;
}
else
{
i++;
}
free(ArrayName);
}
else
Expand Down Expand Up @@ -658,24 +714,6 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen,
VarRec->PerWriterBlockStart[WriterRank + 1] =
VarRec->PerWriterBlockStart[WriterRank] + BlockCount;
}
if (VarRec->MinMaxOffset != SIZE_MAX)
{
MinMaxStruct MinMax;
MinMax.Init(VarRec->Type);
for (size_t B = 0; B < BlockCount; B++)
{
void *MMs = *(void **)(((char *)meta_base) +
VarRec->MinMaxOffset);
char *BlockMinAddr =
(((char *)MMs) + 2 * B * VarRec->ElementSize);
char *BlockMaxAddr =
(((char *)MMs) + (2 * B + 1) * VarRec->ElementSize);
ApplyElementMinMax(MinMax, VarRec->Type,
(void *)BlockMinAddr);
ApplyElementMinMax(MinMax, VarRec->Type,
(void *)BlockMaxAddr);
}
}
}
}
else
Expand Down Expand Up @@ -1121,7 +1159,7 @@ BP5Deserializer::GenerateReadRequests(const bool doAllocTempBuffers,
RR.Timestep = Req->Step;
RR.WriterRank = WriterRank;
RR.StartOffset =
writer_meta_base->DataLocation[NeededBlock];
writer_meta_base->DataBlockLocation[NeededBlock];

RR.ReadLength =
helper::GetDataTypeSize(Req->VarRec->Type) *
Expand Down Expand Up @@ -1179,9 +1217,9 @@ BP5Deserializer::GenerateReadRequests(const bool doAllocTempBuffers,
RR.Timestep = Req->Step;
RR.WriterRank = WriterRank;
RR.StartOffset =
writer_meta_base->DataLocation[Block];
writer_meta_base->DataBlockLocation[Block];
RR.ReadLength =
writer_meta_base->DataLengths[Block];
writer_meta_base->DataBlockSize[Block];
RR.DestinationAddr = nullptr;
if (doAllocTempBuffers)
{
Expand Down Expand Up @@ -1228,7 +1266,7 @@ BP5Deserializer::GenerateReadRequests(const bool doAllocTempBuffers,
RR.Timestep = Req->Step;
RR.WriterRank = WriterRank;
RR.StartOffset =
writer_meta_base->DataLocation[Block] +
writer_meta_base->DataBlockLocation[Block] +
StartOffsetInBlock;
RR.ReadLength =
EndOffsetInBlock - StartOffsetInBlock;
Expand Down Expand Up @@ -1292,7 +1330,7 @@ void BP5Deserializer::FinalizeGet(const ReadRequest &Read, const bool freeAddr)
std::lock_guard<std::mutex> lockGuard(mutexDecompress);
core::Decompress(IncomingData,
((MetaArrayRecOperator *)writer_meta_base)
->DataLengths[Read.BlockID],
->DataBlockSize[Read.BlockID],
decompressBuffer.data());
}
IncomingData = decompressBuffer.data();
Expand Down
Loading