libs: Bring out LZ4 from rdkafka and update it to v1.9.4

.gitmodules

@@ -154,3 +154,6 @@
 [submodule "osquery/experimental/experiments/linuxevents/libraries/linuxevents/src"]
 	path = osquery/experimental/experiments/linuxevents/libraries/linuxevents/src
 	url = https://github.com/trailofbits/linuxevents
+[submodule "libraries/cmake/source/lz4/src"]
+	path = libraries/cmake/source/lz4/src
+	url = https://github.com/lz4/lz4.git

CMakeLists.txt

@@ -86,6 +86,8 @@ endfunction()
 
 function(importLibraries)
   set(library_descriptor_list
+    # Leave lz4 at the top (see LZ4 CMakeLists.txt)
+    "Linux,Darwin,Windows:lz4"
     "Linux,Darwin:augeas"
     "Linux,Darwin,Windows:boost"
     "Linux,Darwin,Windows:bzip2"

libraries/cmake/source/librdkafka/CMakeLists.txt

@@ -6,6 +6,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only)
 
 function(librdkafkaMain)
+
   set(library_root "${CMAKE_CURRENT_SOURCE_DIR}/src")
 
   if(PLATFORM_LINUX)
@@ -26,9 +27,6 @@ function(librdkafkaMain)
 
   add_library(thirdparty_rdkafka_c
     "${library_root}/src/crc32c.c"
-    "${library_root}/src/lz4.c"
-    "${library_root}/src/lz4frame.c"
-    "${library_root}/src/lz4hc.c"
     "${library_root}/src/rdaddr.c"
     "${library_root}/src/rdavl.c"
     "${library_root}/src/rdbuf.c"
@@ -112,8 +110,12 @@ function(librdkafkaMain)
     )
   endif()
 
+  set(LZ4_NAMESPACE "RDKAFKA")
+  renameLZ4APIs("RDKAFKA" "lz4_frame_renames")
+
   target_compile_definitions(thirdparty_rdkafka_c PRIVATE
-    XXH_NAMESPACE=RDKAFKA_
+    XXH_NAMESPACE="RDKAFKA_"
+    ${lz4_frame_renames}
   )
 
   target_include_directories(thirdparty_rdkafka_c PRIVATE
@@ -148,8 +150,8 @@ function(librdkafkaMain)
   target_link_libraries(thirdparty_rdkafka_c
     PRIVATE
       thirdparty_c_settings
-
-    PUBLIC
+      thirdparty_lz4
+      thirdparty_lz4_frame
       thirdparty_openssl
       thirdparty_zlib
       thirdparty_zstd

libraries/cmake/source/lz4/CMakeLists.txt

@@ -0,0 +1,94 @@
+# Copyright (c) 2014-present, The osquery authors
+#
+# This source code is licensed as defined by the LICENSE file found in the
+# root directory of this source tree.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only)
+
+# rdkafka currently embeds a copy of LZ4 sources, but rdkafka doesn't have a CPE
+# to search for CVEs.
+# For this reason and because we normally prefer to build all the needed libraries
+# separately, we brought out LZ4, so that we can track its CVEs and update it independently.
+# One problem with this though is that LZ4 depends on xxhash and rdkafka uses
+# a slightly modified version of xxhash APIs.
+# We therefore have to build a copy of LZ4 for each user, and we have to namespace the APIs
+# for each dependee (rdkafka currently), so that they also refer to the specific namespaced version
+# of the xxhash APIs.
+
+# This helper function preprends a namespace to the exported LZ4 frame APIs,
+# which are the ones that depend on xxhash.
+# It should be accessible to all the other libraries,
+# since we have put the lz4 library as the first processed one in the main CMakeLists.txt.
+function(renameLZ4APIs namespace output_var)
+
+  # NOTE: This list has to be manually kept up to date
+  list(APPEND lz4_frame_apis
+    "LZ4F_isError"
+    "LZ4F_getErrorName"
+    "LZ4F_compressionLevel_max"
+    "LZ4F_compressFrameBound"
+    "LZ4F_compressFrame"
+    "LZ4F_getVersion"
+    "LZ4F_createCompressionContext"
+    "LZ4F_freeCompressionContext"
+    "LZ4F_compressBegin"
+    "LZ4F_compressBound"
+    "LZ4F_compressUpdate"
+    "LZ4F_flush"
+    "LZ4F_compressEnd"
+    "LZ4F_createDecompressionContext"
+    "LZ4F_freeDecompressionContext"
+    "LZ4F_headerSize"
+    "LZ4F_getFrameInfo"
+    "LZ4F_decompress"
+    "LZ4F_resetDecompressionContext"
+    "LZ4F_getErrorCode"
+    "LZ4F_getBlockSize"
+    "LZ4F_uncompressedUpdate"
+    "LZ4F_createCDict"
+    "LZ4F_freeCDict"
+    "LZ4F_compressFrame_usingCDict"
+    "LZ4F_compressBegin_usingCDict"
+    "LZ4F_decompress_usingDict"
+    "LZ4F_decompress_usingDict"
+    "LZ4F_createCompressionContext_advanced"
+    "LZ4F_createDecompressionContext_advanced"
+    "LZ4F_createCDict_advanced"
+  )
+
+  foreach(api ${lz4_frame_apis})
+    list(APPEND ${output_var} ${api}=${namespace}_${api})
+  endforeach()
+
+  set(${output_var} "${${output_var}}" PARENT_SCOPE)
+endfunction()
+
+function(lz4Main)
+
+set(library_root "${CMAKE_CURRENT_SOURCE_DIR}/src")
+
+add_library(thirdparty_lz4
+  "${library_root}/lib/lz4.c"
+  "${library_root}/lib/lz4hc.c"
+  "${library_root}/lib/lz4.h"
+  "${library_root}/lib/lz4hc.h"
+)
+
+add_library(thirdparty_lz4_frame INTERFACE)
+target_sources(thirdparty_lz4_frame INTERFACE
+  "${library_root}/lib/lz4frame.c"
+  "${library_root}/lib/lz4frame.h"
+)
+
+target_link_libraries(thirdparty_lz4_frame INTERFACE
+  thirdparty_lz4
+)
+
+target_link_libraries(thirdparty_lz4
+  PRIVATE
+    thirdparty_c_settings
+)
+
+endfunction()
+
+lz4Main()

libraries/cmake/source/lz4/README.md

@@ -0,0 +1,5 @@
+# LZ4 library build notes
+
+## All platforms
+
+Just look at the CMakeLists.txt file under `build\cmake`

libraries/cmake/source/modules/Findlz4.cmake

@@ -0,0 +1,15 @@
+# Copyright (c) 2014-present, The osquery authors
+#
+# This source code is licensed as defined by the LICENSE file found in the
+# root directory of this source tree.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only)
+
+include("${CMAKE_CURRENT_LIST_DIR}/utils.cmake")
+
+importSourceSubmodule(
+  NAME "lz4"
+
+  SHALLOW_SUBMODULES
+    "src"
+)

libraries/third_party_libraries_manifest.json

@@ -320,5 +320,12 @@
     "version": "1.4.0",
     "commit": "83b51e9f886be7c2a4d477b6e7bc6db831791d8d",
     "ignored-cves": ["CVE-2021-24031"]
+  },
+  "lz4": {
+    "product": "lz4",
+    "vendor": "lz4_project",
+    "version": "1.9.4",
+    "commit": "5ff839680134437dbf4678f3d0c7b371d84f4964",
+    "ignored-cves": []
   }
 }