Compression: Refactor to support multiple compression (#8790)

ref #8789
pingcap · Mar 19, 2024 · 63e83b3 · 63e83b3
1 parent f061a58
commit 63e83b3
Show file tree

Hide file tree

Showing 31 changed files with 2,024 additions and 645 deletions.
diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt
@@ -263,8 +263,11 @@ if (NOT USE_INTERNAL_ZSTD_LIBRARY)
 endif ()
 
 if (USE_QPL)
-    target_link_libraries (dbms ${QPL_LIBRARY})
-    target_include_directories (dbms PRIVATE ${QPL_INCLUDE_DIR})
+    target_link_libraries (tiflash_common_io PUBLIC ${QPL_LIBRARY})
+    target_include_directories (dbms BEFORE PRIVATE ${QPL_INCLUDE_DIR})
+    set_source_files_properties (
+        src/IO/Compression/CompressionCodecDeflateQpl.cpp
+        PROPERTIES COMPILE_FLAGS "-mwaitpkg")
 endif ()
 
 target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR})

diff --git a/dbms/src/Flash/Coprocessor/CHBlockChunkCodecV1.cpp b/dbms/src/Flash/Coprocessor/CHBlockChunkCodecV1.cpp
@@ -18,6 +18,8 @@
 #include <DataTypes/DataTypeNullable.h>
 #include <Flash/Coprocessor/CHBlockChunkCodecV1.h>
 #include <IO/Buffer/ReadBufferFromString.h>
+#include <IO/Compression/CompressionFactory.h>
+#include <IO/Compression/CompressionInfo.h>
 
 namespace DB
 {
@@ -554,7 +556,9 @@ CHBlockChunkCodecV1::EncodeRes CHBlockChunkCodecV1::encode(std::string_view str,
     assert(compression_method != CompressionMethod::NONE);
 
     String compressed_buffer;
-    size_t compressed_size = CompressionEncode(str, CompressionSettings(compression_method), compressed_buffer);
+    auto codec = CompressionFactory::create(CompressionSettings(compression_method));
+    compressed_buffer.resize(codec->getCompressedReserveSize(str.size()));
+    size_t compressed_size = codec->compress(str.data(), str.size(), compressed_buffer.data());
     compressed_buffer.resize(compressed_size);
     return compressed_buffer;
 }

diff --git a/dbms/src/Flash/Coprocessor/CHBlockChunkCodecV1.h b/dbms/src/Flash/Coprocessor/CHBlockChunkCodecV1.h
@@ -16,8 +16,8 @@
 
 #include <Flash/Coprocessor/CHBlockChunkCodec.h>
 #include <IO/Compression/CompressedReadBuffer.h>
-#include <IO/Compression/CompressedStream.h>
 #include <IO/Compression/CompressedWriteBuffer.h>
+#include <IO/Compression/CompressionMethod.h>
 
 namespace DB
 {

diff --git a/dbms/src/Flash/Coprocessor/ChunkDecodeAndSquash.cpp b/dbms/src/Flash/Coprocessor/ChunkDecodeAndSquash.cpp
@@ -15,6 +15,7 @@
 #include <Flash/Coprocessor/CHBlockChunkCodecV1.h>
 #include <Flash/Coprocessor/ChunkDecodeAndSquash.h>
 #include <IO/Buffer/ReadBufferFromString.h>
+#include <IO/Compression/CompressionInfo.h>
 
 namespace DB
 {

diff --git a/dbms/src/Flash/Coprocessor/tests/gtest_block_chunk_codec.cpp b/dbms/src/Flash/Coprocessor/tests/gtest_block_chunk_codec.cpp
@@ -16,6 +16,7 @@
 #include <Flash/Coprocessor/CHBlockChunkCodecV1.h>
 #include <IO/Buffer/ReadBufferFromString.h>
 #include <TestUtils/ColumnGenerator.h>
+#include <TestUtils/TiFlashTestBasic.h>
 #include <gtest/gtest.h>
 
 
@@ -69,6 +70,7 @@ void test_enocde_release_data(VecCol && batch_columns, const Block & header, con
 }
 
 TEST(CHBlockChunkCodec, ChunkCodecV1)
+try
 {
     size_t block_num = 10;
     size_t rows = 10;
@@ -217,14 +219,15 @@ TEST(CHBlockChunkCodec, ChunkCodecV1)
         ASSERT_FALSE(source_str.empty());
         ASSERT_EQ(static_cast<CompressionMethodByte>(source_str[0]), CompressionMethodByte::NONE);
 
-        for (auto mode : {CompressionMethod::LZ4, CompressionMethod::ZSTD})
+        for (const auto method : {CompressionMethod::LZ4, CompressionMethod::ZSTD})
         {
-            auto compressed_str_a = CHBlockChunkCodecV1::encode({&source_str[1], source_str.size() - 1}, mode);
-            auto compressed_str_b = CHBlockChunkCodecV1{header}.encode(blocks.front(), mode);
+            auto compressed_str_a = CHBlockChunkCodecV1::encode({&source_str[1], source_str.size() - 1}, method);
+            auto compressed_str_b = CHBlockChunkCodecV1{header}.encode(blocks.front(), method);
 
             ASSERT_EQ(compressed_str_a, compressed_str_b);
         }
     }
 }
+CATCH
 
 } // namespace DB::tests
diff --git a/dbms/src/Flash/Mpp/MPPTunnelSetHelper.cpp b/dbms/src/Flash/Mpp/MPPTunnelSetHelper.cpp
@@ -15,6 +15,7 @@
 #include <Flash/Coprocessor/CHBlockChunkCodec.h>
 #include <Flash/Coprocessor/CHBlockChunkCodecV1.h>
 #include <Flash/Mpp/MPPTunnelSetHelper.h>
+#include <IO/Compression/CompressionInfo.h>
 
 namespace DB::MPPTunnelSetHelper
 {

diff --git a/dbms/src/Flash/Mpp/MPPTunnelSetWriter.h b/dbms/src/Flash/Mpp/MPPTunnelSetWriter.h
@@ -15,7 +15,7 @@
 #pragma once
 
 #include <Flash/Mpp/MPPTunnelSet.h>
-#include <IO/Compression/CompressedStream.h>
+#include <IO/Compression/CompressionMethod.h>
 
 namespace DB
 {

diff --git a/dbms/src/IO/Compression/CodecDeflateQpl.cpp b/dbms/src/IO/Compression/CodecDeflateQpl.cpp