Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

nvcomp tests: update hardcoded lengths #156

Merged
merged 4 commits into from
Jan 26, 2023
Merged

nvcomp tests: update hardcoded lengths #156

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3a8f39a
nvcomp tests: update hardcoded lengths
madsbk Jan 23, 2023
bcd7f22
Use approx. when comparing compression size
madsbk Jan 23, 2023
46ef2df
doc
madsbk Jan 23, 2023
9632651
cleanup
madsbk Jan 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 45 additions & 32 deletions python/tests/test_nvcomp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,26 @@

np = pytest.importorskip("numpy")
cupy = pytest.importorskip("cupy")
cudf = pytest.importorskip("cudf")
kvikio = pytest.importorskip("kvikio")
libnvcomp = pytest.importorskip("kvikio.nvcomp")


# TODO: don't hardcode the following expected values
LEN = {
"ANS": 11144,
"Bitcomp": 3208,
"Cascaded": 600,
"Gdeflate": 760,
"LZ4": 393,
"Snappy": 3548,
}


def assert_compression_size(actual, desired, rtol=0.1):
"""Compression ratios might change slightly between library versions"""
np.testing.assert_allclose(actual, desired, rtol=rtol)


def managers():
return [
libnvcomp.ANSManager,
Expand Down Expand Up @@ -70,7 +85,7 @@ def test_ans_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.ANSManager(**inputs)
final = compressor.compress(data)
assert len(final) == 11144
assert_compression_size(len(final), LEN["ANS"])


@pytest.mark.parametrize(
Expand All @@ -93,7 +108,7 @@ def test_bitcomp_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.BitcompManager(**inputs)
final = compressor.compress(data)
assert len(final) == 3208
assert_compression_size(len(final), LEN["Bitcomp"])


@pytest.mark.parametrize(
Expand All @@ -104,7 +119,7 @@ def test_bitcomp_inputs(inputs):
{"algo": 1},
{"algo": 2},
],
[3208, 3208, 3208],
[LEN["Bitcomp"], LEN["Bitcomp"], LEN["Bitcomp"]],
),
)
def test_bitcomp_algorithms(inputs, expected):
Expand All @@ -113,7 +128,7 @@ def test_bitcomp_algorithms(inputs, expected):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.BitcompManager(**inputs)
final = compressor.compress(data)
assert len(final) == expected
assert_compression_size(len(final), expected)


@pytest.mark.parametrize(
Expand Down Expand Up @@ -167,7 +182,7 @@ def test_cascaded_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.CascadedManager(**inputs)
final = compressor.compress(data)
assert len(final) == 600
assert_compression_size(len(final), LEN["Cascaded"])


@pytest.mark.parametrize(
Expand All @@ -192,7 +207,7 @@ def test_gdeflate_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.GdeflateManager(**inputs)
final = compressor.compress(data)
assert len(final) == 732
assert_compression_size(len(final), LEN["Gdeflate"])


@pytest.mark.parametrize(
Expand All @@ -201,7 +216,7 @@ def test_gdeflate_inputs(inputs):
[
{"algo": 0},
],
[732],
[LEN["Gdeflate"]],
),
)
def test_gdeflate_algorithms(inputs, expected):
Expand All @@ -210,20 +225,21 @@ def test_gdeflate_algorithms(inputs, expected):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.GdeflateManager(**inputs)
final = compressor.compress(data)
assert len(final) == expected
assert_compression_size(len(final), expected)


@pytest.mark.xfail(raises=ValueError)
@pytest.mark.parametrize(
"inputs, expected", zip([{"algo": 1}, {"algo": 2}], [732, 732])
"inputs, expected",
zip([{"algo": 1}, {"algo": 2}], [LEN["Gdeflate"], LEN["Gdeflate"]]),
)
def test_gdeflate_algorithms_not_implemented(inputs, expected):
size = 10000
dtype = np.int8
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.GdeflateManager(**inputs)
final = compressor.compress(data)
assert len(final) == expected
assert_compression_size(len(final), expected)


@pytest.mark.parametrize(
Expand All @@ -248,7 +264,7 @@ def test_lz4_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.LZ4Manager(**inputs)
final = compressor.compress(data)
assert len(final) == 393
assert_compression_size(len(final), LEN["LZ4"])


@pytest.mark.parametrize(
Expand All @@ -268,40 +284,40 @@ def test_snappy_inputs(inputs):
data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype))
compressor = libnvcomp.SnappyManager(**inputs)
final = compressor.compress(data)
assert len(final) == 3548
assert_compression_size(len(final), LEN["Snappy"])


@pytest.mark.parametrize(
"compressor_size",
zip(
managers(),
[
{
{ # ANS
"max_compressed_buffer_size": 89373,
"num_chunks": 1,
"uncompressed_buffer_size": 10000,
},
{
{ # Bitcomp
"max_compressed_buffer_size": 16432,
"num_chunks": 1,
"uncompressed_buffer_size": 10000,
},
{
{ # Cascaded
"max_compressed_buffer_size": 12460,
"num_chunks": 3,
"uncompressed_buffer_size": 10000,
},
{
{ # Gdeflate
"max_compressed_buffer_size": 131160,
"num_chunks": 1,
"uncompressed_buffer_size": 10000,
},
{
{ # LZ4
"max_compressed_buffer_size": 65888,
"num_chunks": 1,
"uncompressed_buffer_size": 10000,
},
{
{ # Snappy
"max_compressed_buffer_size": 76575,
"num_chunks": 1,
"uncompressed_buffer_size": 10000,
Expand All @@ -323,6 +339,10 @@ def test_get_compression_config_with_default_options(compressor_size):
)
compressor_instance = compressor()
result = compressor_instance.configure_compression(len(data))
assert_compression_size(
result.pop("max_compressed_buffer_size"),
expected.pop("max_compressed_buffer_size"),
)
assert result == expected


Expand Down Expand Up @@ -373,6 +393,9 @@ def test_get_decompression_config_with_default_options(manager, expected):
result = compressor_instance.configure_decompression_with_compressed_buffer(
compressed
)
assert_compression_size(
result.pop("decomp_data_size"), expected.pop("decomp_data_size")
)
assert result == expected


Expand Down Expand Up @@ -427,22 +450,12 @@ def test_get_required_scratch_buffer_size(manager, expected):
compressor_instance = manager()
compressor_instance.configure_compression(len(data))
buffer_size = compressor_instance.get_required_scratch_buffer_size()
assert buffer_size == expected
assert_compression_size(buffer_size, expected)


@pytest.mark.parametrize(
"manager, expected",
zip(
managers(),
[
11144, # ANS
3208, # Bitcomp
600, # Cascaded
732, # Gdeflate
393, # LZ4
3548, # Snappy
],
),
zip(managers(), list(LEN.values())),
)
def test_get_compressed_output_size(manager, expected):
length = 10000
Expand All @@ -457,7 +470,7 @@ def test_get_compressed_output_size(manager, expected):
compressor_instance = manager()
compressed = compressor_instance.compress(data)
buffer_size = compressor_instance.get_compressed_output_size(compressed)
assert buffer_size == expected
assert_compression_size(buffer_size, expected)


@pytest.mark.parametrize("manager", managers())
Expand Down