[COMPARE ALL CHANGES] [encryption] Manage file key by the file to encrypt itself

.github/workflows/jobs-linux-run-tests.yml

@@ -78,13 +78,24 @@ jobs:
     - uses: "./.github/actions/pre-steps"
     - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j5 && ctest -j5
     - uses: "./.github/actions/post-steps"
-  build-linux-encrypted_env-no_compression:
+  build-linux-encrypted_env-no_compression-no_openssl:
     runs-on: ubuntu-latest
     container:
       image: zjay437/rocksdb:0.6
     steps:
     - uses: actions/[email protected]
     - uses: "./.github/actions/pre-steps"
-    - run: mkdir build && cd build && cmake -DWITH_SNAPPY=0 -DWITH_ZLIB=0 -DWITH_BZ2=0 -DWITH_LZ4=0 -DWITH_ZSTD=0 .. && make V=1 -j5 && ctest -j5 -V
+    - run: mkdir build && cd build && cmake -DWITH_OPENSSL=0 -DWITH_SNAPPY=0 -DWITH_ZLIB=0 -DWITH_BZ2=0 -DWITH_LZ4=0 -DWITH_ZSTD=0 .. && make V=1 -j5 && ctest -j5 -V
     - run: "cd build/tools && ./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression'"
     - uses: "./.github/actions/post-steps"
+  build-linux-encrypted_env-openssl:
+    runs-on: ubuntu-latest
+    container:
+      image: zjay437/rocksdb:0.6
+    steps:
+    - uses: actions/[email protected]
+    - uses: "./.github/actions/pre-steps"
+    - run: |
+       export ENCRYPTED_ENV=AES
+       mkdir build && cd build && cmake -DWITH_OPENSSL=1 .. && make V=1 -j5 && ctest -j5 -V
+    - uses: "./.github/actions/post-steps"

CMakeLists.txt

@@ -70,6 +70,7 @@ option(WITH_SNAPPY "build with SNAPPY" OFF)
 option(WITH_LZ4 "build with lz4" OFF)
 option(WITH_ZLIB "build with zlib" OFF)
 option(WITH_ZSTD "build with zstd" OFF)
+option(WITH_OPENSSL "build with openssl" OFF)
 option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, regardles of the system code page" OFF)
 if (WITH_WINDOWS_UTF8_FILENAMES)
   add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
@@ -174,6 +175,14 @@ else()
     include_directories(${ZSTD_INCLUDE_DIR})
     list(APPEND THIRDPARTY_LIBS zstd::zstd)
   endif()
+
+  if(WITH_OPENSSL)
+    find_package(OpenSSL REQUIRED)
+    add_definitions(-DOPENSSL)
+    include_directories(${OPENSSL_INCLUDE_DIR})
+    # Only the crypto library is needed.
+    list(APPEND THIRDPARTY_LIBS ${OPENSSL_CRYPTO_LIBRARIES})
+  endif()
 endif()
 
 option(WITH_MD_LIBRARY "build with MD" ON)
@@ -736,6 +745,7 @@ set(SOURCES
         db/write_controller.cc
         db/write_stall_stats.cc
         db/write_thread.cc
+        encryption/encryption.cc
         env/composite_env.cc
         env/env.cc
         env/env_chroot.cc
@@ -1375,6 +1385,7 @@ if(WITH_TESTS)
         db/write_batch_test.cc
         db/write_callback_test.cc
         db/write_controller_test.cc
+        encryption/encryption_test.cc
         env/env_test.cc
         env/io_posix_test.cc
         env/mock_env_test.cc

Makefile

@@ -703,6 +703,7 @@ TESTS_PLATFORM_DEPENDENT := \
 	crc32c_test \
 	coding_test \
 	inlineskiplist_test \
+	encryption_test \
 	env_basic_test \
 	env_test \
 	env_logger_test \
@@ -1981,6 +1982,9 @@ cache_reservation_manager_test: $(OBJ_DIR)/cache/cache_reservation_manager_test.
 wide_column_serialization_test: $(OBJ_DIR)/db/wide/wide_column_serialization_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+encryption_test: encryption/encryption_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 #-------------------------------------------------
 # make install related stuff
 PREFIX ?= /usr/local

TARGETS

@@ -108,6 +108,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/write_controller.cc",
         "db/write_stall_stats.cc",
         "db/write_thread.cc",
+        "encryption/encryption.cc",
         "env/composite_env.cc",
         "env/env.cc",
         "env/env_chroot.cc",

build_tools/build_detect_platform

@@ -478,6 +478,19 @@ EOF
         fi
     fi
 
+    if ! test $ROCKSDB_DISABLE_OPENSSL; then
+        # Test whether OpenSSL library is installed
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+          #include <openssl/crypto.h>
+          int main() {}
+EOF
+        if [ "$?" = 0 ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -DOPENSSL"
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lcrypto"
+            JAVA_LDFLAGS="$JAVA_LDFLAGS -lcrypto"
+        fi
+    fi
+
     if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then
         # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
         $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null  <<EOF