Add zstd vendoring

Signed-off-by: Petr Shumilov <[email protected]>

.github/workflows/Build.yml

@@ -146,7 +146,7 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install [email protected] re2c cmake coreutils libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
+        brew install [email protected] re2c cmake coreutils libiconv re2 pcre yaml-cpp googletest shivammathur/php/[email protected]
         brew link --overwrite --force shivammathur/php/[email protected]
         /opt/homebrew/opt/[email protected]/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/[email protected]/libexec/bin/pip install --break-system-packages jsonschema
 

.github/workflows/Dockerfile.buster

@@ -23,7 +23,7 @@ RUN apt-get update && \
       python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \
     pip3 install -r /tmp/requirements.txt && \
     apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-      libzstd-dev libyaml-cpp-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
+      libyaml-cpp-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
       libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* libnuma-dev composer && \
     rm -rf /var/lib/apt/lists/* && \
     update-alternatives --set php /usr/bin/php7.4

.github/workflows/Dockerfile.focal

@@ -16,7 +16,7 @@ RUN apt-get update && \
       python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \
     python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \
     apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-      libzstd-dev libyaml-cpp-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
+      libyaml-cpp-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
       libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* libnuma-dev composer unzip && \
     rm -rf /var/lib/apt/lists/*
 

.github/workflows/Dockerfile.jammy

@@ -19,7 +19,7 @@ RUN apt update && \
       build-essential devscripts fakeroot git cmake make g++ lld gperf netcat patch \
       python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools && \
     apt install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-      libzstd-dev libyaml-cpp-dev zlib1g-dev php7.4-dev libnuma-dev unzip  && \
+      libyaml-cpp-dev zlib1g-dev php7.4-dev libnuma-dev unzip  && \
     # Install MySQL-related and PostgreSQL-related libraries from the Ubuntu 20.04 (Focal Fossa) repository.
     # Ubuntu 22.04 has moved to OpenSSL 3.0; however, KPHP relies on OpenSSL 1.1.1.
     # Therefore, we use downgraded libraries that are compatible with OpenSSL 1.1.1.

.github/workflows/macos.yml

@@ -33,7 +33,7 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install [email protected] re2c cmake coreutils libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
+        brew install [email protected] re2c cmake coreutils libiconv re2 pcre yaml-cpp googletest shivammathur/php/[email protected]
         brew link --overwrite --force shivammathur/php/[email protected]
         /opt/homebrew/opt/[email protected]/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/[email protected]/libexec/bin/pip install --break-system-packages jsonschema
       

.gitmodules

@@ -19,3 +19,6 @@
 	path = third-party/nghttp2
 	url = https://github.com/VKCOM/nghttp2.git
 	branch = buster
+[submodule "third-party/zstd"]
+	path = third-party/zstd
+	url = https://github.com/VKCOM/libzstd.git

CMakeLists.txt

@@ -33,6 +33,7 @@ include(cmake/popular-common.cmake)
 # Common third-parties build
 include(${THIRD_PARTY_DIR}/zlib-cmake/zlib.cmake)
 include(${THIRD_PARTY_DIR}/openssl-cmake/openssl.cmake)
+include(${THIRD_PARTY_DIR}/zstd-cmake/zstd.cmake)
 
 # TODO: use FetchContent_Declare instead of include_directories
 include_directories(.)
@@ -123,7 +124,7 @@ install(FILES ${AUTO_DIR}/runtime/runtime-headers.h
         DESTINATION ${INSTALL_KPHP_SOURCE}/objs/generated/auto/runtime/)
 
 set(CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS "vk-flex-data, libuber-h3, libpcre3-dev, libre2-dev, libyaml-cpp-dev, \
-                                       libzstd-dev, g++, kphp-timelib, libnuma-dev")
+                                       g++, kphp-timelib, libnuma-dev")
 
 if (PDO_DRIVER_MYSQL)
     set(CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS "${CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS}, libmysqlclient-dev")

common/common-tests.cmake

@@ -19,8 +19,7 @@ prepend(COMMON_TESTS_SOURCES ${COMMON_DIR}/
         wrappers/string_view-test.cpp
         ucontext/ucontext-portable-test.cpp)
 
-prepare_cross_platform_libs(COMMON_TESTS_LIBS zstd)
-set(COMMON_TESTS_LIBS vk::common_src vk::net_src vk::binlog_src vk::unicode ${COMMON_TESTS_LIBS} ${EPOLL_SHIM_LIB} OpenSSL::Crypto)
+set(COMMON_TESTS_LIBS vk::common_src vk::net_src vk::binlog_src vk::unicode ${COMMON_TESTS_LIBS} ${EPOLL_SHIM_LIB} OpenSSL::Crypto ZSTD::zstd)
 if(COMPILE_RUNTIME_LIGHT)
     set(COMMON_TESTS_LIBS ${COMMON_TESTS_LIBS} ZLIB::ZLIB_PIC)
 else()

common/common.cmake

@@ -64,10 +64,12 @@ if(COMPILER_CLANG)
 endif()
 
 vk_add_library(common_src OBJECT ${COMMON_ALL_SOURCES})
+add_dependencies(common_src openssl zstd)
+target_include_directories(common_src PUBLIC ${ZSTD_INCLUDE_DIRS})
 if(COMPILE_RUNTIME_LIGHT)
+    add_dependencies(common_src zlib-pic)
     target_include_directories(common_src PUBLIC ${ZLIB_PIC_INCLUDE_DIRS})
-    add_dependencies(common_src openssl zlib-pic)
 else()
+    add_dependencies(common_src zlib-no-pic)
     target_include_directories(common_src PUBLIC ${ZLIB_NO_PIC_INCLUDE_DIRS})
-    add_dependencies(common_src openssl zlib-no-pic)
 endif()

common/kfs/kfs.cpp

@@ -18,7 +18,7 @@
 #include <time.h>
 #include <unistd.h>
 #include "zlib/zlib.h"
-#include <zstd.h>
+#include "zstd/zstd.h"
 
 #include "common/binlog/kdb-binlog-common.h"
 #include "common/binlog/snapshot-shifts.h"

common/tl/methods/rwm.cpp

@@ -5,7 +5,7 @@
 #include "common/tl/methods/rwm.h"
 
 #include <memory>
-#include <zstd.h>
+#include "zstd/zstd.h"
 
 #include "common/algorithms/arithmetic.h"
 #include "common/tl/methods/compression.h"

compiler/compiler-settings.cpp

@@ -368,7 +368,7 @@ void CompilerSettings::init() {
   ld_flags.value_ = extra_ld_flags.get();
   append_curl(cxx_default_flags, ld_flags.value_, third_party_path);
   append_apple_options(cxx_default_flags, ld_flags.value_);
-  std::vector<vk::string_view> system_installed_static_libs{"pcre", "re2", "yaml-cpp", "h3", "zstd", "kphp-timelib"};
+  std::vector<vk::string_view> system_installed_static_libs{"pcre", "re2", "yaml-cpp", "h3", "kphp-timelib"};
 
 #ifdef KPHP_TIMELIB_LIB_DIR
   ld_flags.value_ += " -L" KPHP_TIMELIB_LIB_DIR;
@@ -424,14 +424,15 @@ void CompilerSettings::init() {
   system_installed_dynamic_libs.emplace_back("rt");
 #endif
 
+  append_3dparty_lib(ld_flags.value_, third_party_path, "ssl");
+  append_3dparty_lib(ld_flags.value_, third_party_path, "crypto");
+  append_3dparty_lib(ld_flags.value_, third_party_path, "nghttp2");
+  append_3dparty_lib(ld_flags.value_, third_party_path, "zstd");
   if (is_k2_mode) {
     append_3dparty_lib(ld_flags.value_, third_party_path, "z-pic");
   } else {
     append_3dparty_lib(ld_flags.value_, third_party_path, "z-no-pic");
   }
-  append_3dparty_lib(ld_flags.value_, third_party_path, "ssl");
-  append_3dparty_lib(ld_flags.value_, third_party_path, "crypto");
-  append_3dparty_lib(ld_flags.value_, third_party_path, "nghttp2");
 
   append_if_doesnt_contain(ld_flags.value_, system_installed_dynamic_libs, "-l");
   ld_flags.value_ += " -rdynamic";

docs/kphp-internals/developing-and-extending-kphp/compiling-kphp-from-sources.md

@@ -54,7 +54,7 @@ Install packages
 apt-get update
 apt install git cmake-data=3.16* cmake=3.16* make g++ gperf python3-minimal python3-jsonschema \
             curl-kphp-vk libuber-h3-dev kphp-timelib libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-            libzstd-dev libyaml-cpp-dev php7.4-dev libmysqlclient-dev libnuma-dev \
+            libyaml-cpp-dev php7.4-dev libmysqlclient-dev libnuma-dev \
             postgresql postgresql-server-dev-all libpq-dev libldap-dev libkrb5-dev
 ```
 
@@ -74,7 +74,7 @@ Install packages
 apt-get update
 apt install git cmake make g++ gperf python3-minimal python3-jsonschema \
             curl-kphp-vk libuber-h3-dev kphp-timelib libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-            libzstd-dev libyaml-cpp-dev php7.4-dev libmysqlclient-dev libnuma-dev \
+            libyaml-cpp-dev php7.4-dev libmysqlclient-dev libnuma-dev \
             postgresql postgresql-server-dev-all libpq-dev libldap-dev libkrb5-dev
 ```
 
@@ -85,7 +85,7 @@ Make sure you have `brew` and `clang` (at least `Apple clang version 10.0.0`)
 # Install dependencies
 brew tap shivammathur/php
 brew update
-brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
+brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp googletest shivammathur/php/[email protected]
 brew link --overwrite shivammathur/php/[email protected]
 pip3 install jsonschema
 

net/net-tests.cmake

@@ -4,8 +4,7 @@ prepend(NET_TESTS_SOURCES ${BASE_DIR}/net/
         net-test.cpp
         time-slice-test.cpp)
 
-prepare_cross_platform_libs(NET_TESTS_LIBS zstd)
-set(NET_TESTS_LIBS vk::common_src vk::net_src vk::binlog_src vk::unicode ${NET_TESTS_LIBS} ${EPOLL_SHIM_LIB} OpenSSL::Crypto)
+set(NET_TESTS_LIBS vk::common_src vk::net_src vk::binlog_src vk::unicode ${NET_TESTS_LIBS} ${EPOLL_SHIM_LIB} OpenSSL::Crypto ZSTD::zstd)
 if(COMPILE_RUNTIME_LIGHT)
     set(NET_TESTS_LIBS ${NET_TESTS_LIBS} ZLIB::ZLIB_PIC)
 else()

runtime/runtime.cmake

@@ -140,13 +140,13 @@ allow_deprecated_declarations(${BASE_DIR}/runtime/allocator.cpp ${BASE_DIR}/runt
 allow_deprecated_declarations_for_apple(${BASE_DIR}/runtime/inter-process-mutex.cpp)
 
 vk_add_library(kphp_runtime OBJECT ${KPHP_RUNTIME_ALL_SOURCES})
-target_include_directories(kphp_runtime PUBLIC ${BASE_DIR} ${OPENSSL_INCLUDE_DIR} ${ZLIB_NO_PIC_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
+target_include_directories(kphp_runtime PUBLIC ${BASE_DIR} ${OPENSSL_INCLUDE_DIR} ${ZLIB_NO_PIC_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIRS})
 
-add_dependencies(kphp_runtime kphp-timelib curl)
+add_dependencies(kphp_runtime kphp-timelib curl zstd)
 add_dependencies(curl openssl zlib-no-pic)
 
-prepare_cross_platform_libs(RUNTIME_LIBS yaml-cpp re2 zstd h3) # todo: linking between static libs is no-op, is this redundant? do we need to add mysqlclient here?
-set(RUNTIME_LIBS vk::kphp_runtime vk::kphp_server vk::runtime-common vk::popular_common vk::unicode vk::common_src vk::binlog_src vk::net_src ${RUNTIME_LIBS} CURL::curl OpenSSL::SSL OpenSSL::Crypto m ZLIB::ZLIB_NO_PIC pthread)
+prepare_cross_platform_libs(RUNTIME_LIBS yaml-cpp re2 h3) # todo: linking between static libs is no-op, is this redundant? do we need to add mysqlclient here?
+set(RUNTIME_LIBS vk::kphp_runtime vk::kphp_server vk::runtime-common vk::popular_common vk::unicode vk::common_src vk::binlog_src vk::net_src ${RUNTIME_LIBS} CURL::curl OpenSSL::SSL OpenSSL::Crypto m ZLIB::ZLIB_NO_PIC ZSTD::zstd pthread)
 vk_add_library(kphp-full-runtime STATIC)
 target_link_libraries(kphp-full-runtime PUBLIC ${RUNTIME_LIBS})
 set_target_properties(kphp-full-runtime PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR})

runtime/zstd.cpp

@@ -4,7 +4,7 @@
 
 #define ZSTD_STATIC_LINKING_ONLY
 
-#include <zstd.h>
+#include "zstd/zstd.h"
 
 #include "common/smart_ptrs/unique_ptr_with_delete_function.h"
 #include "runtime-common/stdlib/string/string-context.h"

tests/cpp/runtime/zstd-test.cpp

@@ -1,6 +1,6 @@
 #include <gtest/gtest.h>
 
-#include <zstd.h>
+#include "zstd/zstd.h"
 
 #include "runtime-common/stdlib/string/string-context.h"
 

third-party/curl-cmake/curl.cmake

@@ -84,6 +84,5 @@ set_target_properties(CURL::curl PROPERTIES
 
 # Set variables indicating that curl has been installed
 set(CURL_FOUND ON)
-set(CURL_LIBRARIES ${CURL_INSTALL_DIR}/lib/libcurl.a)
 
 cmake_print_variables(CURL_LIBRARIES)

third-party/nghttp2-cmake/nghttp2.cmake

@@ -68,8 +68,8 @@ ExternalProject_Add(
             COMMAND ${CMAKE_COMMAND} --build ${NGHTTP2_BUILD_DIR} --config $<CONFIG> -j
         INSTALL_COMMAND
             COMMAND ${CMAKE_COMMAND} --install ${NGHTTP2_BUILD_DIR} --prefix ${NGHTTP2_INSTALL_DIR} --config $<CONFIG>
-            COMMAND ${CMAKE_COMMAND} -E copy_directory ${NGHTTP2_INSTALL_DIR}/include ${INCLUDE_DIR}
-            COMMAND ${CMAKE_COMMAND} -E copy ${NGHTTP2_INSTALL_DIR}/lib/libnghttp2.a ${LIB_DIR}
+            COMMAND ${CMAKE_COMMAND} -E copy_directory ${NGHTTP2_INCLUDE_DIRS} ${INCLUDE_DIR}
+            COMMAND ${CMAKE_COMMAND} -E copy ${NGHTTP2_LIBRARIES} ${LIB_DIR}
         BUILD_IN_SOURCE 0
 )
 

third-party/zstd-cmake/zstd.cmake

@@ -0,0 +1,63 @@
+update_git_submodule(${THIRD_PARTY_DIR}/zstd)
+
+set(ZSTD_SOURCE_DIR     ${THIRD_PARTY_DIR}/zstd)
+set(ZSTD_BUILD_DIR      ${CMAKE_BINARY_DIR}/third-party/zstd/build)
+set(ZSTD_INSTALL_DIR    ${CMAKE_BINARY_DIR}/third-party/zstd/install)
+set(ZSTD_BINARY_DIR     ${ZSTD_BUILD_DIR}/lib)
+set(ZSTD_LIBRARIES      ${ZSTD_INSTALL_DIR}/lib/libzstd.a)
+set(ZSTD_INCLUDE_DIRS   ${ZSTD_INSTALL_DIR}/include)
+set(ZSTD_PATCH_DIR      ${ZSTD_BUILD_DIR}/debian/patches/)
+set(ZSTD_PATCH_SERIES   ${ZSTD_BUILD_DIR}/debian/patches/series)
+# Ensure the build, installation and "include" directories exists
+file(MAKE_DIRECTORY ${ZSTD_BUILD_DIR})
+file(MAKE_DIRECTORY ${ZSTD_INSTALL_DIR})
+file(MAKE_DIRECTORY ${ZSTD_INCLUDE_DIRS})
+
+# The configuration has been based on:
+# https://sources.debian.org/src/libzstd/1.3.8%2Bdfsg-3%2Bdeb10u2/debian/rules/
+set(ZSTD_COMPILE_FLAGS "$ENV{CFLAGS} -g0 -fno-pic")
+if(NOT APPLE)
+    set(ZSTD_COMPILE_FLAGS "${ZSTD_COMPILE_FLAGS} -static")
+endif()
+
+set(ZSTD_MAKE_ARGS
+        CC=${CMAKE_C_COMPILER}
+        CFLAGS=${OPENSSL_COMPILE_FLAGS}
+)
+
+set(ZSTD_MAKE_INSTALL_ARGS
+        PREFIX=${ZSTD_INSTALL_DIR}
+        INCLUDEDIR=${ZSTD_INSTALL_DIR}/include/zstd
+)
+
+ExternalProject_Add(
+        zstd
+        PREFIX ${ZSTD_BUILD_DIR}
+        SOURCE_DIR ${ZSTD_SOURCE_DIR}
+        INSTALL_DIR ${ZSTD_INSTALL_DIR}
+        BINARY_DIR ${ZSTD_BINARY_DIR}
+        BUILD_BYPRODUCTS ${ZSTD_INSTALL_DIR}/lib/libzstd.a
+        PATCH_COMMAND
+            COMMAND ${CMAKE_COMMAND} -E copy_directory ${ZSTD_SOURCE_DIR} ${ZSTD_BUILD_DIR}
+            COMMAND ${CMAKE_COMMAND} -DBUILD_DIR=${ZSTD_BUILD_DIR} -DPATCH_SERIES=${ZSTD_PATCH_SERIES} -DPATCH_DIR=${ZSTD_PATCH_DIR} -P ../../cmake/apply_patches.cmake
+        CONFIGURE_COMMAND
+            COMMAND # Nothing to configure
+        BUILD_COMMAND
+            COMMAND ${CMAKE_COMMAND} -E env ${ZSTD_MAKE_ARGS} make libzstd.a -j
+        INSTALL_COMMAND
+            COMMAND ${CMAKE_COMMAND} -E env ${ZSTD_MAKE_INSTALL_ARGS} make install-static install-includes
+            COMMAND ${CMAKE_COMMAND} -E copy_directory ${ZSTD_INCLUDE_DIRS} ${INCLUDE_DIR}
+            COMMAND ${CMAKE_COMMAND} -E copy ${ZSTD_LIBRARIES} ${LIB_DIR}
+        BUILD_IN_SOURCE 0
+)
+
+add_library(ZSTD::zstd STATIC IMPORTED)
+set_target_properties(ZSTD::zstd PROPERTIES
+        IMPORTED_LOCATION ${ZSTD_LIBRARIES}
+        INTERFACE_INCLUDE_DIRECTORIES ${ZSTD_INCLUDE_DIRS}
+)
+
+# Set variables indicating that zstd has been installed
+set(ZSTD_FOUND ON)
+
+cmake_print_variables(ZSTD_LIBRARIES)