From a35d9e7c4b583ba0f499020aa16edc5818111518 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Thu, 26 Jan 2023 15:18:22 +0100 Subject: [PATCH] nvcomp tests: update hardcoded lengths (#156) @thomcom can we avoid hard coding the expected compression lengths? Authors: - Mads R. B. Kristensen (https://github.com/madsbk) Approvers: - Benjamin Zaitlen (https://github.com/quasiben) URL: https://github.com/rapidsai/kvikio/pull/156 --- python/tests/test_nvcomp.py | 77 ++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/python/tests/test_nvcomp.py b/python/tests/test_nvcomp.py index 5b2bd8c0fa..207ee83bee 100644 --- a/python/tests/test_nvcomp.py +++ b/python/tests/test_nvcomp.py @@ -5,11 +5,26 @@ np = pytest.importorskip("numpy") cupy = pytest.importorskip("cupy") -cudf = pytest.importorskip("cudf") kvikio = pytest.importorskip("kvikio") libnvcomp = pytest.importorskip("kvikio.nvcomp") +# TODO: don't hardcode the following expected values +LEN = { + "ANS": 11144, + "Bitcomp": 3208, + "Cascaded": 600, + "Gdeflate": 760, + "LZ4": 393, + "Snappy": 3548, +} + + +def assert_compression_size(actual, desired, rtol=0.1): + """Compression ratios might change slightly between library versions""" + np.testing.assert_allclose(actual, desired, rtol=rtol) + + def managers(): return [ libnvcomp.ANSManager, @@ -70,7 +85,7 @@ def test_ans_inputs(inputs): data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype)) compressor = libnvcomp.ANSManager(**inputs) final = compressor.compress(data) - assert len(final) == 11144 + assert_compression_size(len(final), LEN["ANS"]) @pytest.mark.parametrize( @@ -93,7 +108,7 @@ def test_bitcomp_inputs(inputs): data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype)) compressor = libnvcomp.BitcompManager(**inputs) final = compressor.compress(data) - assert len(final) == 3208 + assert_compression_size(len(final), LEN["Bitcomp"]) @pytest.mark.parametrize( @@ -104,7 +119,7 @@ def test_bitcomp_inputs(inputs): {"algo": 1}, {"algo": 2}, ], - [3208, 3208, 3208], + [LEN["Bitcomp"], LEN["Bitcomp"], LEN["Bitcomp"]], ), ) def test_bitcomp_algorithms(inputs, expected): @@ -113,7 +128,7 @@ def test_bitcomp_algorithms(inputs, expected): data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype)) compressor = libnvcomp.BitcompManager(**inputs) final = compressor.compress(data) - assert len(final) == expected + assert_compression_size(len(final), expected) @pytest.mark.parametrize( @@ -167,7 +182,7 @@ def test_cascaded_inputs(inputs): data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype)) compressor = libnvcomp.CascadedManager(**inputs) final = compressor.compress(data) - assert len(final) == 600 + assert_compression_size(len(final), LEN["Cascaded"]) @pytest.mark.parametrize( @@ -192,7 +207,7 @@ def test_gdeflate_inputs(inputs): data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype)) compressor = libnvcomp.GdeflateManager(**inputs) final = compressor.compress(data) - assert len(final) == 732 + assert_compression_size(len(final), LEN["Gdeflate"]) @pytest.mark.parametrize( @@ -201,7 +216,7 @@ def test_gdeflate_inputs(inputs): [ {"algo": 0}, ], - [732], + [LEN["Gdeflate"]], ), ) def test_gdeflate_algorithms(inputs, expected): @@ -210,12 +225,13 @@ def test_gdeflate_algorithms(inputs, expected): data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype)) compressor = libnvcomp.GdeflateManager(**inputs) final = compressor.compress(data) - assert len(final) == expected + assert_compression_size(len(final), expected) @pytest.mark.xfail(raises=ValueError) @pytest.mark.parametrize( - "inputs, expected", zip([{"algo": 1}, {"algo": 2}], [732, 732]) + "inputs, expected", + zip([{"algo": 1}, {"algo": 2}], [LEN["Gdeflate"], LEN["Gdeflate"]]), ) def test_gdeflate_algorithms_not_implemented(inputs, expected): size = 10000 @@ -223,7 +239,7 @@ def test_gdeflate_algorithms_not_implemented(inputs, expected): data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype)) compressor = libnvcomp.GdeflateManager(**inputs) final = compressor.compress(data) - assert len(final) == expected + assert_compression_size(len(final), expected) @pytest.mark.parametrize( @@ -248,7 +264,7 @@ def test_lz4_inputs(inputs): data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype)) compressor = libnvcomp.LZ4Manager(**inputs) final = compressor.compress(data) - assert len(final) == 393 + assert_compression_size(len(final), LEN["LZ4"]) @pytest.mark.parametrize( @@ -268,7 +284,7 @@ def test_snappy_inputs(inputs): data = cupy.array(np.arange(0, size // dtype(0).itemsize, dtype=dtype)) compressor = libnvcomp.SnappyManager(**inputs) final = compressor.compress(data) - assert len(final) == 3548 + assert_compression_size(len(final), LEN["Snappy"]) @pytest.mark.parametrize( @@ -276,32 +292,32 @@ def test_snappy_inputs(inputs): zip( managers(), [ - { + { # ANS "max_compressed_buffer_size": 89373, "num_chunks": 1, "uncompressed_buffer_size": 10000, }, - { + { # Bitcomp "max_compressed_buffer_size": 16432, "num_chunks": 1, "uncompressed_buffer_size": 10000, }, - { + { # Cascaded "max_compressed_buffer_size": 12460, "num_chunks": 3, "uncompressed_buffer_size": 10000, }, - { + { # Gdeflate "max_compressed_buffer_size": 131160, "num_chunks": 1, "uncompressed_buffer_size": 10000, }, - { + { # LZ4 "max_compressed_buffer_size": 65888, "num_chunks": 1, "uncompressed_buffer_size": 10000, }, - { + { # Snappy "max_compressed_buffer_size": 76575, "num_chunks": 1, "uncompressed_buffer_size": 10000, @@ -323,6 +339,10 @@ def test_get_compression_config_with_default_options(compressor_size): ) compressor_instance = compressor() result = compressor_instance.configure_compression(len(data)) + assert_compression_size( + result.pop("max_compressed_buffer_size"), + expected.pop("max_compressed_buffer_size"), + ) assert result == expected @@ -373,6 +393,9 @@ def test_get_decompression_config_with_default_options(manager, expected): result = compressor_instance.configure_decompression_with_compressed_buffer( compressed ) + assert_compression_size( + result.pop("decomp_data_size"), expected.pop("decomp_data_size") + ) assert result == expected @@ -427,22 +450,12 @@ def test_get_required_scratch_buffer_size(manager, expected): compressor_instance = manager() compressor_instance.configure_compression(len(data)) buffer_size = compressor_instance.get_required_scratch_buffer_size() - assert buffer_size == expected + assert_compression_size(buffer_size, expected) @pytest.mark.parametrize( "manager, expected", - zip( - managers(), - [ - 11144, # ANS - 3208, # Bitcomp - 600, # Cascaded - 732, # Gdeflate - 393, # LZ4 - 3548, # Snappy - ], - ), + zip(managers(), list(LEN.values())), ) def test_get_compressed_output_size(manager, expected): length = 10000 @@ -457,7 +470,7 @@ def test_get_compressed_output_size(manager, expected): compressor_instance = manager() compressed = compressor_instance.compress(data) buffer_size = compressor_instance.get_compressed_output_size(compressed) - assert buffer_size == expected + assert_compression_size(buffer_size, expected) @pytest.mark.parametrize("manager", managers())