Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

warn users when compressed buffer is larger than original data #2914

Closed
wants to merge 3 commits into from
Closed

warn users when compressed buffer is larger than original data #2914

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cd4ab70
warn users when compressed buffer is larger than original data
JasonRuonanWang Oct 19, 2021
7f7ec57
fix png
JasonRuonanWang Oct 19, 2021
23d9adc
unified variable name
JasonRuonanWang Oct 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions source/adios2/operator/compress/CompressBZIP2.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ CompressBZIP2::CompressBZIP2(const Params &parameters)
}

size_t CompressBZIP2::Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, DataType type,
const Dims &blockCount, DataType dataType,
char *bufferOut, const Params &parameters)
{

Expand All @@ -47,7 +47,7 @@ size_t CompressBZIP2::Compress(const char *dataIn, const Dims &blockStart,
// Universal operator metadata end

const size_t sizeIn =
helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type));
helper::GetTotalSize(blockCount, helper::GetDataTypeSize(dataType));
const size_t batches = sizeIn / DefaultMaxFileBatchSize + 1;

// bzip2 V1 metadata
Expand All @@ -61,7 +61,7 @@ size_t CompressBZIP2::Compress(const char *dataIn, const Dims &blockStart,
if (!parameters.empty())
{
const std::string hint(" in call to CompressBZIP2 Compress " +
ToString(type) + "\n");
ToString(dataType) + "\n");
helper::SetParameterValueInt("blockSize100k", parameters, blockSize100k,
hint);
helper::SetParameterValueInt("verbosity", parameters, verbosity, hint);
Expand Down Expand Up @@ -112,6 +112,14 @@ size_t CompressBZIP2::Compress(const char *dataIn, const Dims &blockStart,
destOffset += destLen;
}

if (sizeIn < destOffset)
{
std::cerr
<< "ADIOS2 bzip2 operator warning: compression buffer size larger "
"than original data size. Consider disabling compression."
<< std::endl;
}

return destOffset;
}

Expand Down Expand Up @@ -199,7 +207,10 @@ size_t CompressBZIP2::Decompress(const char *bufferIn, const size_t sizeIn,
return 0;
}

bool CompressBZIP2::IsDataTypeValid(const DataType type) const { return true; }
bool CompressBZIP2::IsDataTypeValid(const DataType dataType) const
{
return true;
}

void CompressBZIP2::CheckStatus(const int status, const std::string hint) const
{
Expand Down
20 changes: 16 additions & 4 deletions source/adios2/operator/compress/CompressBlosc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ CompressBlosc::CompressBlosc(const Params &parameters)
}

size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, const DataType type,
const Dims &blockCount, const DataType dataType,
char *bufferOut, const Params &parameters)
{
size_t bufferOutOffset = 0;
Expand All @@ -56,7 +56,7 @@ size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart,
// Universal operator metadata end

const size_t sizeIn =
helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type));
helper::GetTotalSize(blockCount, helper::GetDataTypeSize(dataType));

// blosc V1 metadata
PutParameter(bufferOut, bufferOutOffset, sizeIn);
Expand Down Expand Up @@ -157,7 +157,7 @@ size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart,
*headerPtr = DataHeader{};
bufferOutOffset += sizeof(DataHeader);

int32_t typesize = helper::GetDataTypeSize(type);
int32_t typesize = helper::GetDataTypeSize(dataType);
if (typesize > BLOSC_MAX_TYPESIZE)
typesize = 1;

Expand Down Expand Up @@ -226,6 +226,15 @@ size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart,
}

blosc_destroy();

if (sizeIn < bufferOutOffset)
{
std::cerr
<< "ADIOS2 blosc operator warning: compression buffer size larger "
"than original data size. Consider disabling compression."
<< std::endl;
}

return bufferOutOffset;
}

Expand Down Expand Up @@ -301,7 +310,10 @@ size_t CompressBlosc::Decompress(const char *bufferIn, const size_t sizeIn,
return 0;
}

bool CompressBlosc::IsDataTypeValid(const DataType type) const { return true; }
bool CompressBlosc::IsDataTypeValid(const DataType dataType) const
{
return true;
}

size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn,
const size_t sizeIn,
Expand Down
27 changes: 19 additions & 8 deletions source/adios2/operator/compress/CompressLibPressio.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,8 +286,9 @@ CompressLibPressio::CompressLibPressio(const Params &parameters)
}

size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, const DataType type,
char *bufferOut, const Params &parameters)
const Dims &blockCount,
const DataType dataType, char *bufferOut,
const Params &parameters)
{
const uint8_t bufferVersion = 1;
size_t bufferOutOffset = 0;
Expand All @@ -306,7 +307,7 @@ size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart,
{
PutParameter(bufferOut, bufferOutOffset, d);
}
PutParameter(bufferOut, bufferOutOffset, type);
PutParameter(bufferOut, bufferOutOffset, dataType);
PutParameter(bufferOut, bufferOutOffset,
static_cast<uint8_t>(pressio_major_version()));
PutParameter(bufferOut, bufferOutOffset,
Expand All @@ -318,7 +319,7 @@ size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart,

auto inputs_dims = adios_to_libpressio_dims(blockCount);
pressio_data *input_buf = pressio_data_new_nonowning(
adios_to_libpressio_dtype(type), const_cast<char *>(dataIn),
adios_to_libpressio_dtype(dataType), const_cast<char *>(dataIn),
inputs_dims.size(), inputs_dims.data());
pressio_data *output_buf =
pressio_data_new_empty(pressio_byte_dtype, 0, nullptr);
Expand Down Expand Up @@ -350,6 +351,16 @@ size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart,
pressio_data_free(input_buf);
pressio_data_free(output_buf);

const size_t sizeIn =
helper::GetTotalSize(blockCount, helper::GetDataTypeSize(dataType));
if (sizeIn < bufferOutOffset)
{
std::cerr
<< "ADIOS2 LibPressio operator warning: compression buffer size "
"larger than original data size. Consider disabling compression."
<< std::endl;
}

return bufferOutOffset;
}

Expand All @@ -369,7 +380,7 @@ size_t CompressLibPressio::DecompressV1(const char *bufferIn,
{
blockCount[i] = GetParameter<size_t, size_t>(bufferIn, bufferInOffset);
}
const DataType type = GetParameter<DataType>(bufferIn, bufferInOffset);
const DataType dataType = GetParameter<DataType>(bufferIn, bufferInOffset);
m_VersionInfo =
" Data is compressed using LibPressio Version " +
std::to_string(GetParameter<uint8_t>(bufferIn, bufferInOffset)) + "." +
Expand All @@ -380,7 +391,7 @@ size_t CompressLibPressio::DecompressV1(const char *bufferIn,

std::vector<size_t> dims = adios_to_libpressio_dims(blockCount);
pressio_data *output_buf = pressio_data_new_owning(
adios_to_libpressio_dtype(type), dims.size(), dims.data());
adios_to_libpressio_dtype(dataType), dims.size(), dims.data());

size_t newSizeIn = sizeIn - bufferInOffset;
pressio_data *input_buf = pressio_data_new_nonowning(
Expand Down Expand Up @@ -443,10 +454,10 @@ size_t CompressLibPressio::Decompress(const char *bufferIn, const size_t sizeIn,
return 0;
}

bool CompressLibPressio::IsDataTypeValid(const DataType type) const
bool CompressLibPressio::IsDataTypeValid(const DataType dataType) const
{
#define declare_type(T) \
if (helper::GetDataType<T>() == type) \
if (helper::GetDataType<T>() == dataType) \
{ \
return true; \
}
Expand Down
26 changes: 18 additions & 8 deletions source/adios2/operator/compress/CompressMGARD.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ CompressMGARD::CompressMGARD(const Params &parameters)
}

size_t CompressMGARD::Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, const DataType type,
const Dims &blockCount, const DataType dataType,
char *bufferOut, const Params &parameters)
{
const uint8_t bufferVersion = 1;
Expand All @@ -47,7 +47,7 @@ size_t CompressMGARD::Compress(const char *dataIn, const Dims &blockStart,
{
PutParameter(bufferOut, bufferOutOffset, d);
}
PutParameter(bufferOut, bufferOutOffset, type);
PutParameter(bufferOut, bufferOutOffset, dataType);
PutParameter(bufferOut, bufferOutOffset,
static_cast<uint8_t>(MGARD_VERSION_MAJOR));
PutParameter(bufferOut, bufferOutOffset,
Expand All @@ -64,7 +64,7 @@ size_t CompressMGARD::Compress(const char *dataIn, const Dims &blockStart,

// set type
int mgardType = -1;
if (type == helper::GetDataType<double>())
if (dataType == helper::GetDataType<double>())
{
mgardType = 1;
}
Expand Down Expand Up @@ -123,6 +123,16 @@ size_t CompressMGARD::Compress(const char *dataIn, const Dims &blockStart,

bufferOutOffset += sizeOut;

const size_t sizeIn =
helper::GetTotalSize(blockCount, helper::GetDataTypeSize(dataType));
if (sizeIn < bufferOutOffset)
{
std::cerr
<< "ADIOS2 mgard operator warning: compression buffer size larger "
"than original data size. Consider disabling compression."
<< std::endl;
}

return bufferOutOffset;
}

Expand All @@ -142,7 +152,7 @@ size_t CompressMGARD::DecompressV1(const char *bufferIn, const size_t sizeIn,
{
blockCount[i] = GetParameter<size_t, size_t>(bufferIn, bufferInOffset);
}
const DataType type = GetParameter<DataType>(bufferIn, bufferInOffset);
const DataType dataType = GetParameter<DataType>(bufferIn, bufferInOffset);
m_VersionInfo =
" Data is compressed using MGARD Version " +
std::to_string(GetParameter<uint8_t>(bufferIn, bufferInOffset)) + "." +
Expand All @@ -152,7 +162,7 @@ size_t CompressMGARD::DecompressV1(const char *bufferIn, const size_t sizeIn,

int mgardType = -1;

if (type == helper::GetDataType<double>())
if (dataType == helper::GetDataType<double>())
{
mgardType = 1;
}
Expand All @@ -179,7 +189,7 @@ size_t CompressMGARD::DecompressV1(const char *bufferIn, const size_t sizeIn,
static_cast<int>(sizeIn - bufferInOffset), r[0], r[1], r[2], 0.0);

const size_t sizeOut =
helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type));
helper::GetTotalSize(blockCount, helper::GetDataTypeSize(dataType));

std::memcpy(dataOut, dataPtr, sizeOut);

Expand Down Expand Up @@ -215,10 +225,10 @@ size_t CompressMGARD::Decompress(const char *bufferIn, const size_t sizeIn,
return 0;
}

bool CompressMGARD::IsDataTypeValid(const DataType type) const
bool CompressMGARD::IsDataTypeValid(const DataType dataType) const
{
#define declare_type(T) \
if (helper::GetDataType<T>() == type) \
if (helper::GetDataType<T>() == dataType) \
{ \
return true; \
}
Expand Down
19 changes: 16 additions & 3 deletions source/adios2/operator/compress/CompressPNG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ CompressPNG::CompressPNG(const Params &parameters) : Operator("png", parameters)
}

size_t CompressPNG::Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, const DataType type,
const Dims &blockCount, const DataType dataType,
char *bufferOut, const Params &parameters)
{
size_t bufferOutOffset = 0;
Expand Down Expand Up @@ -143,7 +143,7 @@ size_t CompressPNG::Compress(const char *dataIn, const Dims &blockStart,

const uint32_t bytesPerPixel = ndims == 3
? static_cast<uint32_t>(blockCount[2])
: helper::GetDataTypeSize(type);
: helper::GetDataTypeSize(dataType);

const uint32_t width = static_cast<uint32_t>(blockCount[1]);
const uint32_t height = static_cast<uint32_t>(blockCount[0]);
Expand Down Expand Up @@ -188,6 +188,16 @@ size_t CompressPNG::Compress(const char *dataIn, const Dims &blockStart,
PutParameter(bufferOut, paramOffset,
static_cast<uint8_t>(PNG_LIBPNG_VER_RELEASE));

const size_t sizeIn =
helper::GetTotalSize(blockCount, helper::GetDataTypeSize(dataType));
if (sizeIn < destInfo.Offset)
{
std::cerr
<< "ADIOS2 png operator warning: compression buffer size larger "
"than original data size. Consider disabling compression."
<< std::endl;
}

return destInfo.Offset;
}

Expand Down Expand Up @@ -264,7 +274,10 @@ size_t CompressPNG::Decompress(const char *bufferIn, const size_t sizeIn,
return 0;
}

bool CompressPNG::IsDataTypeValid(const DataType type) const { return true; }
bool CompressPNG::IsDataTypeValid(const DataType dataType) const
{
return true;
}

void CompressPNG::CheckStatus(const int status, const std::string hint) const {}

Expand Down
29 changes: 20 additions & 9 deletions source/adios2/operator/compress/CompressSZ.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ namespace compress
CompressSZ::CompressSZ(const Params &parameters) : Operator("sz", parameters) {}

size_t CompressSZ::Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, const DataType varType,
const Dims &blockCount, const DataType dataType,
char *bufferOut, const Params &parameters)
{
const uint8_t bufferVersion = 1;
Expand All @@ -50,15 +50,15 @@ size_t CompressSZ::Compress(const char *dataIn, const Dims &blockStart,
{
PutParameter(bufferOut, bufferOutOffset, d);
}
PutParameter(bufferOut, bufferOutOffset, varType);
PutParameter(bufferOut, bufferOutOffset, dataType);
for (uint8_t i = 0; i < 4; ++i)
{
PutParameter(bufferOut, bufferOutOffset,
static_cast<uint8_t>(versionNumber[i]));
}
// sz V1 metadata end

Dims convertedDims = ConvertDims(blockCount, varType, 4);
Dims convertedDims = ConvertDims(blockCount, dataType, 4);

sz_params sz;
memset(&sz, 0, sizeof(sz_params));
Expand All @@ -82,7 +82,7 @@ size_t CompressSZ::Compress(const char *dataIn, const Dims &blockStart,
static_cast<int>(std::pow(5., static_cast<double>(ndims)));
sz.pwr_type = SZ_PWR_MIN_TYPE;

convertedDims = ConvertDims(blockCount, varType, 4, true, 1);
convertedDims = ConvertDims(blockCount, dataType, 4, true, 1);

/* SZ parameters */
int use_configfile = 0;
Expand Down Expand Up @@ -257,21 +257,21 @@ size_t CompressSZ::Compress(const char *dataIn, const Dims &blockStart,

// Get type info
int dtype = -1;
if (varType == helper::GetDataType<double>() ||
varType == helper::GetDataType<std::complex<double>>())
if (dataType == helper::GetDataType<double>() ||
dataType == helper::GetDataType<std::complex<double>>())
{
dtype = SZ_DOUBLE;
}
else if (varType == helper::GetDataType<float>() ||
varType == helper::GetDataType<std::complex<float>>())
else if (dataType == helper::GetDataType<float>() ||
dataType == helper::GetDataType<std::complex<float>>())
{
dtype = SZ_FLOAT;
}
else
{
throw std::invalid_argument("ERROR: ADIOS2 SZ Compression only support "
"double or float, type: " +
ToString(varType) + " is unsupported\n");
ToString(dataType) + " is unsupported\n");
}

size_t szBufferSize;
Expand All @@ -283,6 +283,17 @@ size_t CompressSZ::Compress(const char *dataIn, const Dims &blockStart,
free(szBuffer);
szBuffer = nullptr;
SZ_Finalize();

const size_t sizeIn =
helper::GetTotalSize(blockCount, helper::GetDataTypeSize(dataType));
if (sizeIn < bufferOutOffset)
{
std::cerr
<< "ADIOS2 sz operator warning: compression buffer size larger "
"than original data size. Consider disabling compression."
<< std::endl;
}

return bufferOutOffset;
}

Expand Down
Loading