Backport 4438 to 2.2.0

.formatignore

@@ -7,6 +7,8 @@ src/lz4frame.c
 src/lz4frame.h
 src/lz4hc.c
 src/lz4hc.h
+src/rdxxhash.c
+src/rdxxhash.h
 src/queue.h
 src/crc32c.c
 src/crc32c.h

.semaphore/semaphore.yml

@@ -2,7 +2,9 @@ version: v1.0
 name: 'librdkafka build and release artifact pipeline'
 agent:
   machine:
-    type: s1-prod-macos-arm64
+    type: s1-prod-macos-13-5-arm64
+execution_time_limit:
+  hours: 3
 global_job_config:
   prologue:
     commands:
@@ -15,7 +17,7 @@ blocks:
     task:
       agent:
         machine:
-          type: s1-prod-macos-arm64
+          type: s1-prod-macos-13-5-arm64
       env_vars:
         - name: ARTIFACT_KEY
           value: p-librdkafka__plat-osx__arch-arm64__lnk-all
@@ -41,7 +43,7 @@ blocks:
     task:
       agent:
         machine:
-          type: s1-prod-macos
+          type: s1-prod-macos-13-5-amd64
       env_vars:
         - name: ARTIFACT_KEY
           value: p-librdkafka__plat-osx__arch-x64__lnk-all
@@ -108,14 +110,23 @@ blocks:
       env_vars:
         - name: CFLAGS
           value: -std=gnu90 # Test minimum C standard, default in CentOS 7
+      prologue:
+        commands:
+          - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
       jobs:
+        - name: 'Build configuration checks'
+          commands:
+            - wget -O rapidjson-dev.deb https://launchpad.net/ubuntu/+archive/primary/+files/rapidjson-dev_1.1.0+dfsg2-3_all.deb
+            - sudo dpkg -i rapidjson-dev.deb
+            - python3 -m pip install -U pip
+            - ./packaging/tools/build-configurations-checks.sh
         - name: 'Build and integration tests'
           commands:
             - wget -O rapidjson-dev.deb https://launchpad.net/ubuntu/+archive/primary/+files/rapidjson-dev_1.1.0+dfsg2-3_all.deb
             - sudo dpkg -i rapidjson-dev.deb
             - python3 -m pip install -U pip
             - python3 -m pip -V
-            - python3 -m pip install -r tests/requirements.txt
+            - (cd tests && python3 -m pip install -r requirements.txt)
             - ./configure --install-deps
             # split these up
             - ./packaging/tools/rdutcoverage.sh
@@ -140,51 +151,40 @@ blocks:
       agent:
         machine:
           type: s1-prod-ubuntu20-04-amd64-2
+      prologue:
+        commands:
+          - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
       epilogue:
         commands:
           - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
       jobs:
-        - name: 'Build: centos6 glibc +gssapi'
-          env_vars:
-            - name: ARTIFACT_KEY
-              value: p-librdkafka__plat-linux__dist-centos6__arch-x64__lnk-std__extra-gssapi
-          commands:
-            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux2010_x86_64 artifacts/librdkafka.tgz
-
-        - name: 'Build: centos6 glibc'
-          env_vars:
-            - name: ARTIFACT_KEY
-              value: p-librdkafka__plat-linux__dist-centos6__arch-x64__lnk-all
-          commands:
-            - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux2010_x86_64 artifacts/librdkafka.tgz
-
-        - name: 'Build: centos7 glibc +gssapi'
+        - name: 'Build: centos8 glibc +gssapi'
           env_vars:
             - name: ARTIFACT_KEY
-              value: p-librdkafka__plat-linux__dist-centos7__arch-x64__lnk-std__extra-gssapi
+              value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi
           commands:
-            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux2014_x86_64 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz
 
-        - name: 'Build: centos7 glibc'
+        - name: 'Build: centos8 glibc'
           env_vars:
             - name: ARTIFACT_KEY
-              value: p-librdkafka__plat-linux__dist-centos7__arch-x64__lnk-all
+              value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-all
           commands:
-            - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux2014_x86_64 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz
 
         - name: 'Build: alpine musl +gssapi'
           env_vars:
             - name: ARTIFACT_KEY
               value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-std__extra-gssapi
           commands:
-            - packaging/tools/build-release-artifacts.sh alpine:3.16 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
 
         - name: 'Build: alpine musl'
           env_vars:
             - name: ARTIFACT_KEY
               value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-all
           commands:
-            - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz
 
 
   - name: 'Linux arm64: release artifact docker builds'
@@ -193,37 +193,40 @@ blocks:
       agent:
         machine:
           type: s1-prod-ubuntu20-04-arm64-1
+      prologue:
+        commands:
+          - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
       epilogue:
         commands:
           - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
       jobs:
-        - name: 'Build: centos7 glibc +gssapi'
+        - name: 'Build: centos8 glibc +gssapi'
           env_vars:
             - name: ARTIFACT_KEY
-              value: p-librdkafka__plat-linux__dist-centos7__arch-arm64__lnk-std__extra-gssapi
+              value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi
           commands:
-            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux2014_aarch64 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
 
-        - name: 'Build: centos7 glibc'
+        - name: 'Build: centos8 glibc'
           env_vars:
             - name: ARTIFACT_KEY
-              value: p-librdkafka__plat-linux__dist-centos7__arch-arm64__lnk-all
+              value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all
           commands:
-            - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux2014_aarch64 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
 
         - name: 'Build: alpine musl +gssapi'
           env_vars:
             - name: ARTIFACT_KEY
               value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi
           commands:
-            - packaging/tools/build-release-artifacts.sh alpine:3.16 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
 
         - name: 'Build: alpine musl'
           env_vars:
             - name: ARTIFACT_KEY
               value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all
           commands:
-            - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16 artifacts/librdkafka.tgz
+            - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz
 
 
   - name: 'Windows x64: MinGW-w64'
@@ -239,11 +242,8 @@ blocks:
           value: UCRT64
       prologue:
         commands:
-          - cache restore msys2-x64-${Env:ARTIFACT_KEY}
           # Set up msys2
           - "& .\\win32\\setup-msys2.ps1"
-          - cache delete msys2-x64-${Env:ARTIFACT_KEY}
-          - cache store msys2-x64-${Env:ARTIFACT_KEY} c:/msys64
       epilogue:
         commands:
           - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ }
@@ -275,25 +275,13 @@ blocks:
       prologue:
         commands:
           # install vcpkg in the parent directory.
-          - pwd
           - cd ..
-          # Restore vcpkg caches, if any.
-          - cache restore vcpkg-archives-$Env:ARTIFACT_KEY
           # Setup vcpkg
           - "& .\\librdkafka\\win32\\setup-vcpkg.ps1"
           - cd librdkafka
           - ..\vcpkg\vcpkg integrate install
           # Install required packages.
           - ..\vcpkg\vcpkg --feature-flags=versions install --triplet $Env:triplet
-          - cd ..
-          - pwd
-          # Store vcpkg caches
-          - ls vcpkg/
-          - echo $Env:VCPKG_ROOT
-          - cache delete vcpkg-archives-$Env:ARTIFACT_KEY
-          - cache store vcpkg-archives-$Env:ARTIFACT_KEY C:/Users/semaphore/AppData/Local/vcpkg/archives
-          - pwd
-          - cd librdkafka
       epilogue:
         commands:
           - Get-ChildItem . -include *.dll -recurse

CHANGELOG.md

@@ -1,3 +1,38 @@
+# librdkafka v2.2.1
+
+*Note: given this patch version contains only a single fix, it's suggested to upgrade to latest backward compatible release instead, as it contains all the issued fixes.
+Following [semver 2.0](https://semver.org/), all our patch and minor releases are backward compatible and our minor releases may also contain fixes.
+Please note that 2.x versions of librdkafka are also backward compatible with 1.x as the major version release was only for the upgrade to OpenSSL 3.x.*
+
+librdkafka v2.2.1 is a maintenance release backporting:
+
+* Fix for idempotent producer fatal errors, triggered after a possibly persisted message state (#4438).
+* Update bundled lz4 (used when `./configure --disable-lz4-ext`) to
+      [v1.9.4](https://github.com/lz4/lz4/releases/tag/v1.9.4), which contains
+      bugfixes and performance improvements (#4726).
+* Upgrade OpenSSL to v3.0.13 (while building from source) with various security fixes,
+     check the [release notes](https://www.openssl.org/news/cl30.txt)
+     (@janjwerner-confluent, #4690).
+* Upgrade zstd to v1.5.6, zlib to v1.3.1, and curl to v8.8.0 (@janjwerner-confluent, #4690).
+* Upgrade Linux dependencies: OpenSSL 3.0.15, CURL 8.10.1 (#4875).
+
+
+
+### Idempotent producer fixes
+
+* After a possibly persisted error, such as a disconnection or a timeout, next expected sequence
+  used to increase, leading to a fatal error if the message wasn't persisted and
+  the second one in queue failed with an `OUT_OF_ORDER_SEQUENCE_NUMBER`.
+  The error could contain the message "sequence desynchronization" with
+  just one possibly persisted error or "rewound sequence number" in case of
+  multiple errored messages.
+  Solved by treating the possible persisted message as _not_ persisted,
+  and expecting a `DUPLICATE_SEQUENCE_NUMBER` error in case it was or
+  `NO_ERROR` in case it wasn't, in both cases the message will be considered
+  delivered (#4438).
+
+
+
 # librdkafka v2.2.0
 
 librdkafka v2.2.0 is a feature release:

LICENSE.lz4

@@ -1,7 +1,7 @@
-src/rdxxhash.[ch] src/lz4*.[ch]: [email protected]:lz4/lz4.git e2827775ee80d2ef985858727575df31fc60f1f3
+src/rdxxhash.[ch] src/lz4*.[ch]: [email protected]:lz4/lz4.git 5ff839680134437dbf4678f3d0c7b371d84f4964
 
 LZ4 Library
-Copyright (c) 2011-2016, Yann Collet
+Copyright (c) 2011-2020, Yann Collet
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

LICENSES.txt

@@ -141,10 +141,10 @@ THE SOFTWARE
 
 LICENSE.lz4
 --------------------------------------------------------------
-src/rdxxhash.[ch] src/lz4*.[ch]: [email protected]:lz4/lz4.git e2827775ee80d2ef985858727575df31fc60f1f3
+src/rdxxhash.[ch] src/lz4*.[ch]: [email protected]:lz4/lz4.git 5ff839680134437dbf4678f3d0c7b371d84f4964
 
 LZ4 Library
-Copyright (c) 2011-2016, Yann Collet
+Copyright (c) 2011-2020, Yann Collet
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

configure.self

@@ -34,7 +34,7 @@ mkl_toggle_option "Development" ENABLE_VALGRIND "--enable-valgrind" "Enable in-c
 
 mkl_toggle_option "Development" ENABLE_REFCNT_DEBUG "--enable-refcnt-debug" "Enable refcnt debugging" "n"
 
-mkl_toggle_option "Feature" ENABLE_LZ4_EXT "--enable-lz4-ext" "Enable external LZ4 library support (builtin version 1.9.3)" "y"
+mkl_toggle_option "Feature" ENABLE_LZ4_EXT "--enable-lz4-ext" "Enable external LZ4 library support (builtin version 1.9.4)" "y"
 mkl_toggle_option "Feature" ENABLE_LZ4_EXT "--enable-lz4" "Deprecated: alias for --enable-lz4-ext" "y"
 
 mkl_toggle_option "Feature" ENABLE_REGEX_EXT "--enable-regex-ext" "Enable external (libc) regex (else use builtin)" "y"

mklove/modules/configure.libcurl

@@ -45,8 +45,8 @@ void foo (void) {
 function install_source {
     local name=$1
     local destdir=$2
-    local ver=7.86.0
-    local checksum="3dfdd39ba95e18847965cd3051ea6d22586609d9011d91df7bc5521288987a82"
+    local ver=8.10.1
+    local checksum="d15ebab765d793e2e96db090f0e172d127859d78ca6f6391d7eafecfd894bbc0"
 
     echo "### Installing $name $ver from source to $destdir"
     if [[ ! -f Makefile ]]; then
@@ -86,8 +86,8 @@ function install_source {
 	--disable-manual \
 	--disable-ldap{,s} \
 	--disable-libcurl-option \
-        --without-{librtmp,libidn2,winidn,nghttp2,nghttp3,ngtcp2,quiche,brotli} &&
-	time make -j &&
+	--without-{librtmp,libidn2,winidn,nghttp2,nghttp3,ngtcp2,quiche,brotli,libpsl} &&
+	time make CPPFLAGS="$CPPFLAGS" -j &&
 	make DESTDIR="${destdir}" prefix=/usr install
     local ret=$?
 

mklove/modules/configure.libssl

@@ -91,8 +91,8 @@ function manual_checks {
 function libcrypto_install_source {
     local name=$1
     local destdir=$2
-    local ver=3.0.8
-    local checksum="6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e"
+    local ver=3.0.15
+    local checksum="23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533"
     local url=https://www.openssl.org/source/openssl-${ver}.tar.gz
 
     local conf_args="--prefix=/usr --openssldir=/usr/lib/ssl no-shared no-zlib"

mklove/modules/configure.libzstd

@@ -42,8 +42,8 @@ void foo (void) {
 function install_source {
     local name=$1
     local destdir=$2
-    local ver=1.5.2
-    local checksum="7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0"
+    local ver=1.5.6
+    local checksum="8c29e06cf42aacc1eafc4077ae2ec6c6fcb96a626157e0593d5e82a34fd403c1"
 
     echo "### Installing $name $ver from source to $destdir"
     if [[ ! -f Makefile ]]; then

mklove/modules/configure.zlib

@@ -42,8 +42,8 @@ void foo (void) {
 function install_source {
     local name=$1
     local destdir=$2
-    local ver=1.2.13
-    local checksum="b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30"
+    local ver=1.3.1
+    local checksum="9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23"
 
     echo "### Installing $name $ver from source to $destdir"
     if [[ ! -f Makefile ]]; then

packaging/cp/README.md

@@ -7,8 +7,7 @@ The base_url is the http S3 bucket path to the a PR job, or similar.
 
 ## How to use
 
-    $ ./verify-packages.sh 5.3 https://thes3bucketpath/X/Y
-
+    $ ./verify-packages.sh 7.6 https://packages.confluent.io
 
 Requires docker and patience.
 

packaging/cp/verify-deb.sh

@@ -12,13 +12,13 @@ if [[ -z $base_url ]]; then
 fi
 
 apt-get update
-apt-get install -y apt-transport-https wget
+apt-get install -y apt-transport-https wget gnupg2 lsb-release
 
 wget -qO - ${base_url}/deb/${cpver}/archive.key | apt-key add -
 
-
+release=$(lsb_release -cs)
 cat >/etc/apt/sources.list.d/Confluent.list <<EOF
-deb [arch=amd64] $base_url/deb/${cpver} stable main
+deb [arch=amd64] $base_url/clients/deb ${release} main
 EOF
 
 apt-get update
@@ -28,7 +28,6 @@ gcc /v/check_features.c -o /tmp/check_features -lrdkafka
 
 /tmp/check_features
 
-# Verify plugins
-apt-get install -y confluent-librdkafka-plugins
-
-/tmp/check_features plugin.library.paths monitoring-interceptor
+# FIXME: publish plugins in newer versions
+# apt-get install -y confluent-librdkafka-plugins
+#/tmp/check_features plugin.library.paths monitoring-interceptor

packaging/cp/verify-packages.sh

@@ -17,11 +17,16 @@ fi
 thisdir="$( cd "$(dirname "$0")" ; pwd -P )"
 
 echo "#### Verifying RPM packages ####"
-docker run -v $thisdir:/v centos:7 /v/verify-rpm.sh $cpver $base_url
+docker run -v $thisdir:/v rockylinux:8 /v/verify-rpm.sh $cpver $base_url
+docker run -v $thisdir:/v rockylinux:9 /v/verify-rpm.sh $cpver $base_url
 rpm_status=$?
 
 echo "#### Verifying Debian packages ####"
-docker run -v $thisdir:/v ubuntu:16.04 /v/verify-deb.sh $cpver $base_url
+docker run -v $thisdir:/v debian:10 /v/verify-deb.sh $cpver $base_url
+docker run -v $thisdir:/v debian:11 /v/verify-deb.sh $cpver $base_url
+docker run -v $thisdir:/v debian:12 /v/verify-deb.sh $cpver $base_url
+docker run -v $thisdir:/v ubuntu:20.04 /v/verify-deb.sh $cpver $base_url
+docker run -v $thisdir:/v ubuntu:22.04 /v/verify-deb.sh $cpver $base_url
 deb_status=$?