Skip to content

Commit

Permalink
nvcomp tests: update hardcoded lengths (#156)
Browse files Browse the repository at this point in the history 
@thomcom can we avoid hard coding the expected compression lengths?

Authors:
  - Mads R. B. Kristensen (https://github.com/madsbk)

Approvers:
  - Benjamin Zaitlen (https://github.com/quasiben)

URL: #156
  • Loading branch information
@madsbk
madsbk authored Jan 26, 2023
1 parent f5cb987 commit a35d9e7
Showing 1 changed file with 45 additions and 32 deletions.
77 changes: 45 additions & 32 deletions python/tests/test_nvcomp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,26 @@

np = pytest.importorskip("numpy")
cupy = pytest.importorskip("cupy")
cudf = pytest.importorskip("cudf")
kvikio = pytest.importorskip("kvikio")
libnvcomp = pytest.importorskip("kvikio.nvcomp")


# TODO: don't hardcode the following expected values
LEN = {
"ANS": 11144,
"Bitcomp": 3208,
"Cascaded": 600,
"Gdeflate": 760,
"LZ4": 393,
"Snappy": 3548,
}


def assert_compression_size(actual, desired, rtol=0.1):
"""Compression ratios might change slightly between library versions"""
np.testing.assert_allclose(actual, desired, rtol=rtol)


def managers():
return [
libnvcomp.ANSManager,
Expand Down Expand Up @@ -70,7 +85,7 @@ def test_ans_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.ANSManager(**inputs)
final = compressor.compress(data)
assert len(final) == 11144
assert_compression_size(len(final), LEN["ANS"])


@pytest.mark.parametrize(
Expand All @@ -93,7 +108,7 @@ def test_bitcomp_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.BitcompManager(**inputs)
final = compressor.compress(data)
assert len(final) == 3208
assert_compression_size(len(final), LEN["Bitcomp"])


@pytest.mark.parametrize(
Expand All @@ -104,7 +119,7 @@ def test_bitcomp_inputs(inputs):
{"algo": 1},
{"algo": 2},
],
[3208, 3208, 3208],
[LEN["Bitcomp"], LEN["Bitcomp"], LEN["Bitcomp"]],
),
)
def test_bitcomp_algorithms(inputs, expected):
Expand All @@ -113,7 +128,7 @@ def test_bitcomp_algorithms(inputs, expected):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.BitcompManager(**inputs)
final = compressor.compress(data)
assert len(final) == expected
assert_compression_size(len(final), expected)


@pytest.mark.parametrize(
Expand Down Expand Up @@ -167,7 +182,7 @@ def test_cascaded_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.CascadedManager(**inputs)
final = compressor.compress(data)
assert len(final) == 600
assert_compression_size(len(final), LEN["Cascaded"])


@pytest.mark.parametrize(
Expand All @@ -192,7 +207,7 @@ def test_gdeflate_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.GdeflateManager(**inputs)
final = compressor.compress(data)
assert len(final) == 732
assert_compression_size(len(final), LEN["Gdeflate"])


@pytest.mark.parametrize(
Expand All @@ -201,7 +216,7 @@ def test_gdeflate_inputs(inputs):
[
{"algo": 0},
],
[732],
[LEN["Gdeflate"]],
),
)
def test_gdeflate_algorithms(inputs, expected):
Expand All @@ -210,20 +225,21 @@ def test_gdeflate_algorithms(inputs, expected):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.GdeflateManager(**inputs)
final = compressor.compress(data)
assert len(final) == expected
assert_compression_size(len(final), expected)


@pytest.mark.xfail(raises=ValueError)
@pytest.mark.parametrize(
"inputs, expected", zip([{"algo": 1}, {"algo": 2}], [732, 732])
"inputs, expected",
zip([{"algo": 1}, {"algo": 2}], [LEN["Gdeflate"], LEN["Gdeflate"]]),
)
def test_gdeflate_algorithms_not_implemented(inputs, expected):
size = 10000
dtype = np.int8
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.GdeflateManager(**inputs)
final = compressor.compress(data)
assert len(final) == expected
assert_compression_size(len(final), expected)


@pytest.mark.parametrize(
Expand All @@ -248,7 +264,7 @@ def test_lz4_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.LZ4Manager(**inputs)
final = compressor.compress(data)
assert len(final) == 393
assert_compression_size(len(final), LEN["LZ4"])


@pytest.mark.parametrize(
Expand All @@ -268,40 +284,40 @@ def test_snappy_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.SnappyManager(**inputs)
final = compressor.compress(data)
assert len(final) == 3548
assert_compression_size(len(final), LEN["Snappy"])


@pytest.mark.parametrize(
"compressor_size",
zip(
managers(),
[
{
{ # ANS
"max_compressed_buffer_size": 89373,
"num_chunks": 1,
"uncompressed_buffer_size": 10000,
},
{
{ # Bitcomp
"max_compressed_buffer_size": 16432,
"num_chunks": 1,
"uncompressed_buffer_size": 10000,
},
{
{ # Cascaded
"max_compressed_buffer_size": 12460,
"num_chunks": 3,
"uncompressed_buffer_size": 10000,
},
{
{ # Gdeflate
"max_compressed_buffer_size": 131160,
"num_chunks": 1,
"uncompressed_buffer_size": 10000,
},
{
{ # LZ4
"max_compressed_buffer_size": 65888,
"num_chunks": 1,
"uncompressed_buffer_size": 10000,
},
{
{ # Snappy
"max_compressed_buffer_size": 76575,
"num_chunks": 1,
"uncompressed_buffer_size": 10000,
Expand All @@ -323,6 +339,10 @@ def test_get_compression_config_with_default_options(compressor_size):
)
compressor_instance = compressor()
result = compressor_instance.configure_compression(len(data))
assert_compression_size(
result.pop("max_compressed_buffer_size"),
expected.pop("max_compressed_buffer_size"),
)
assert result == expected


Expand Down Expand Up @@ -373,6 +393,9 @@ def test_get_decompression_config_with_default_options(manager, expected):
result = compressor_instance.configure_decompression_with_compressed_buffer(
compressed
)
assert_compression_size(
result.pop("decomp_data_size"), expected.pop("decomp_data_size")
)
assert result == expected


Expand Down Expand Up @@ -427,22 +450,12 @@ def test_get_required_scratch_buffer_size(manager, expected):
compressor_instance = manager()
compressor_instance.configure_compression(len(data))
buffer_size = compressor_instance.get_required_scratch_buffer_size()
assert buffer_size == expected
assert_compression_size(buffer_size, expected)


@pytest.mark.parametrize(
"manager, expected",
zip(
managers(),
[
11144, # ANS
3208, # Bitcomp
600, # Cascaded
732, # Gdeflate
393, # LZ4
3548, # Snappy
],
),
zip(managers(), list(LEN.values())),
)
def test_get_compressed_output_size(manager, expected):
length = 10000
Expand All @@ -457,7 +470,7 @@ def test_get_compressed_output_size(manager, expected):
compressor_instance = manager()
compressed = compressor_instance.compress(data)
buffer_size = compressor_instance.get_compressed_output_size(compressed)
assert buffer_size == expected
assert_compression_size(buffer_size, expected)


@pytest.mark.parametrize("manager", managers())
Expand Down

0 comments on commit a35d9e7

Please sign in to comment.