rocksdb: add compression type zstd and lz4 (apache#217)

Former-commit-id: fd79b03bbec5acb93a7195b955c86176c9ba8236 [formerly d0b6530] Former-commit-id: c0e1608ee7777a999d97eb9985c1750eb17b873a
neverchanje · Dec 13, 2018 · 4253540 · 4253540
1 parent d1fdf42
commit 4253540
Show file tree

Hide file tree

Showing 11 changed files with 34 additions and 23 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -8,8 +8,15 @@ os: linux
 compiler:
     - gcc
 
+addons:
+  apt:
+    packages:
+      - libsnappy-dev
+      - liblz4-dev
+
 cache:
     - ccache
+    - apt
 
 before_install:
     - wget https://raw.githubusercontent.com/xiaomi/pegasus-common/master/build-depends.tar.gz
@@ -19,9 +26,9 @@ before_install:
     - cd ..
 
 install:
-    - sudo apt-get -f install
-    - sudo apt-get -y install libsnappy-dev
-    - sudo apt-get -y install libgflags-dev
+    - wget https://github.com/facebook/zstd/archive/v1.3.7.zip; unzip v1.3.7; cd zstd-1.3.7;
+    - mkdir cmake-build; cd cmake-build; cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib -DZSTD_BUILD_PROGRAMS=OFF ../build/cmake; sudo make install -j8;
+    - cd ../..
 
 before_script:
     - cd rdsn/thirdparty

diff --git a/run.sh b/run.sh
@@ -200,7 +200,7 @@ function run_build()
 
     echo "INFO: start build rocksdb..."
     ROCKSDB_BUILD_DIR="$ROOT/rocksdb/build"
-    CMAKE_OPTIONS="-DCMAKE_C_COMPILER=$C_COMPILER -DCMAKE_CXX_COMPILER=$CXX_COMPILER"
+    CMAKE_OPTIONS="-DCMAKE_C_COMPILER=$C_COMPILER -DCMAKE_CXX_COMPILER=$CXX_COMPILER -DWITH_LZ4=ON -DWITH_ZSTD=ON -DWITH_SNAPPY=ON -DWITH_BZ2=OFF"
     if [ "$WARNING_ALL" == "YES" ]
     then
         echo "WARNING_ALL=YES"

diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh
@@ -102,7 +102,8 @@ copy_file `get_system_lib server snappy` ${pack}/bin
 copy_file `get_system_lib server crypto` ${pack}/bin
 copy_file `get_system_lib server ssl` ${pack}/bin
 copy_file `get_system_lib server aio` ${pack}/bin
-copy_file `get_system_lib server bz2` ${pack}/bin
+copy_file `get_system_lib server zstd` ${pack}/bin
+copy_file `get_system_lib server lz4` ${pack}/bin
 
 chmod +x ${pack}/bin/pegasus_* ${pack}/bin/*.sh
 chmod -x ${pack}/bin/lib*

diff --git a/scripts/pack_tools.sh b/scripts/pack_tools.sh
@@ -108,7 +108,8 @@ copy_file `get_system_lib shell snappy` ${pack}/DSN_ROOT/lib/
 copy_file `get_system_lib shell crypto` ${pack}/DSN_ROOT/lib/
 copy_file `get_system_lib shell ssl` ${pack}/DSN_ROOT/lib/
 copy_file `get_system_lib shell aio` ${pack}/DSN_ROOT/lib/
-copy_file `get_system_lib shell bz2` ${pack}/DSN_ROOT/lib/
+copy_file `get_system_lib shell zstd` ${pack}/DSN_ROOT/lib/
+copy_file `get_system_lib shell lz4` ${pack}/DSN_ROOT/lib/
 chmod -x ${pack}/DSN_ROOT/lib/*
 
 mkdir -p ${pack}/scripts

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -41,6 +41,13 @@ link_directories(${PEGASUS_ROOT}/lib)
 set(CMAKE_INSTALL_PREFIX ${PEGASUS_ROOT} CACHE STRING "" FORCE)
 message (STATUS "pegasus Installation directory: CMAKE_INSTALL_PREFIX = " ${CMAKE_INSTALL_PREFIX})
 
+# rocksdb
+set(CMAKE_MODULE_PATH "${PEGASUS_PROJECT_DIR}/rocksdb/cmake/modules")
+find_package(snappy REQUIRED)
+find_package(zstd REQUIRED)
+find_package(lz4 REQUIRED)
+set(ROCKSDB_LINK_LIBRARIES rocksdb ${ZSTD_LIBRARIES} ${LZ4_LIBRARIES} ${SNAPPY_LIBRARIES})
+
 add_subdirectory(base)
 add_subdirectory(reporter)
 add_subdirectory(base/test)

diff --git a/src/geo/bench/CMakeLists.txt b/src/geo/bench/CMakeLists.txt
@@ -19,10 +19,8 @@ set(MY_PROJ_LIBS
         s2
         pegasus_client_static
         fmt
-        rocksdb
-        z
-        bz2
-        snappy)
+        ${ROCKSDB_LINK_LIBRARIES}
+        z)
 
 set(MY_BOOST_PACKAGES system filesystem)
 

diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt
@@ -36,10 +36,8 @@ set(MY_PROJ_LIBS
     PocoJSON
     crypto
     fmt
-    rocksdb
+    ${ROCKSDB_LINK_LIBRARIES}
     z
-    bz2
-    snappy
     rt
     aio
     pthread)

diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp
@@ -181,15 +181,18 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
         "pegasus.server",
         "rocksdb_compression_type",
         "snappy",
-        "rocksdb options.compression, default snappy. Supported: none, snappy.");
+        "rocksdb options.compression, default snappy. Supported: none, snappy, zstd, lz4.");
     if (compression_str == "none") {
         _db_opts.compression = rocksdb::kNoCompression;
     } else if (compression_str == "snappy") {
         _db_opts.compression = rocksdb::kSnappyCompression;
+    } else if (compression_str == "zstd") {
+        _db_opts.compression = rocksdb::kZSTD;
+    } else if (compression_str == "lz4") {
+        _db_opts.compression = rocksdb::kLZ4Compression;
     } else {
         dassert("unsupported compression type: %s", compression_str.c_str());
     }
-
     if (_db_opts.compression != rocksdb::kNoCompression) {
         // only compress levels >= 2
         // refer to ColumnFamilyOptions::OptimizeLevelStyleCompaction()

diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt
@@ -32,10 +32,10 @@ set(MY_PROJ_LIBS
         PocoJSON
         crypto
         fmt
-        rocksdb
+        ${ROCKSDB_LINK_LIBRARIES}
         pegasus.base
         gtest
-        z bz2 snappy rt aio pthread
+        z rt aio pthread
         )
 add_definitions(-DPEGASUS_UNIT_TEST)
 add_definitions(-DENABLE_FAIL)

diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt
@@ -37,10 +37,8 @@ set(MY_PROJ_LIBS
     fmt
     pegasus_geo_lib
     s2
-    rocksdb
+    ${ROCKSDB_LINK_LIBRARIES}
     z
-    bz2
-    snappy
     rt
     aio
     pthread)

diff --git a/src/test/bench_test/CMakeLists.txt b/src/test/bench_test/CMakeLists.txt
@@ -16,10 +16,8 @@ set(MY_PROJ_LIB_PATH "../../../rocksdb/build")
 
 set(MY_PROJ_LIBS
         pegasus_client_static
-        rocksdb
+        ${ROCKSDB_LINK_LIBRARIES}
         z
-        bz2
-        snappy
         gflags)
 
 set(MY_BOOST_PACKAGES system filesystem)
-Original file line number
+Diff line change
@@ Expand Up / @@ -19,10 +19,8 @@ set(MY_PROJ_LIBS @@
             s2
             pegasus_client_static
             fmt
-            rocksdb
-            z
-            bz2
-            snappy)
+            ${ROCKSDB_LINK_LIBRARIES}
+            z)
     set(MY_BOOST_PACKAGES system filesystem)
@@ Expand Down @@