From 8be539bc49bd87313dfe0d040c86e76617a44d7a Mon Sep 17 00:00:00 2001 From: Pranav Rathi <4427674+pranavrth@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:39:31 +0530 Subject: [PATCH 1/3] Added changes for Python 3.13 wheel generation --- .semaphore/semaphore.yml | 62 +++++++++++++++++-- examples/requirements.txt | 2 +- setup.py | 2 +- .../schema_registry/requirements.txt | 2 +- tests/requirements.txt | 2 +- tools/build-manylinux.sh | 6 +- tools/mingw-w64/semaphore_commands.sh | 2 +- tools/wheels/build-wheels.bat | 2 +- tools/wheels/build-wheels.sh | 5 +- 9 files changed, 67 insertions(+), 18 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index ae9b3a13b..62b735f65 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -14,7 +14,7 @@ global_job_config: - checkout - mkdir artifacts blocks: - - name: "Wheels: OSX x64" + - name: "Wheels: OSX x64 - Python 3.6-3.12" run: when: "tag =~ '.*'" dependencies: [] @@ -30,10 +30,33 @@ blocks: jobs: - name: Build commands: - - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse + - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse 2.16.2 - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse - artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/ - - name: "Wheels: OSX arm64" + - name: "Wheels: OSX x64 - Python 3.13" + run: + when: "tag =~ '.*'" + dependencies: [] + task: + agent: + machine: + type: s1-prod-macos-13-5-amd64 + env_vars: + - name: OS_NAME + value: osx + - name: ARCH + value: x64 + - name: CIBW_SKIP + value: cp36-* cp37-* cp-* cp38-* cp39-* cp310-* cp311-* cp312-* + - name: CIBW_ENVIRONMENT_MACOS + value: MACOSX_DEPLOYMENT_TARGET=13 + jobs: + - name: Build + commands: + - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse + - tar -czf wheelhouse-macOS-${ARCH}-py313.tgz wheelhouse + - artifact push workflow wheelhouse-macOS-${ARCH}-py313.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}-py313.tgz/ + - name: "Wheels: OSX arm64 - Python 3.8-3.12" run: when: "tag =~ '.*'" dependencies: [] @@ -51,9 +74,34 @@ blocks: jobs: - name: Build commands: - - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse + - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse 2.16.2 - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse - artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/ + - name: "Wheels: OSX arm64 - Python 3.13" + run: + when: "tag =~ '.*'" + dependencies: [] + task: + agent: + machine: + type: s1-prod-macos-13-5-arm64 + env_vars: + - name: OS_NAME + value: osx + - name: CIBW_ARCHS + value: arm64 + - name: ARCH + value: arm64 + - name: CIBW_SKIP + value: cp38-* cp39-* cp310-* cp311-* cp312-* + - name: CIBW_ENVIRONMENT_MACOS + value: MACOSX_DEPLOYMENT_TARGET=13 + jobs: + - name: Build + commands: + - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse + - tar -czf wheelhouse-macOS-${ARCH}-py313.tgz wheelhouse + - artifact push workflow wheelhouse-macOS-${ARCH}-py313.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}-py313.tgz/ - name: "Wheels: Linux arm64" run: when: "tag =~ '.*'" @@ -206,8 +254,10 @@ blocks: run: when: "tag =~ '.*'" dependencies: - - "Wheels: OSX x64" - - "Wheels: OSX arm64" + - "Wheels: OSX x64 - Python 3.6-3.12" + - "Wheels: OSX x64 - Python 3.13" + - "Wheels: OSX arm64 - Python 3.8-3.12" + - "Wheels: OSX arm64 - Python 3.13" - "Wheels: Linux arm64" - "Wheels: Linux x64" - "Wheels: Windows" diff --git a/examples/requirements.txt b/examples/requirements.txt index 5be5932d9..3f434b94d 100644 --- a/examples/requirements.txt +++ b/examples/requirements.txt @@ -5,6 +5,6 @@ uvicorn six pyrsistent==0.16.1;python_version<"3.0" pyrsistent;python_version>"3.0" -jsonschema +jsonschema<4.18.0 protobuf requests diff --git a/setup.py b/setup.py index fe12df2d6..0b771dc5e 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ JSON_REQUIRES = ['pyrsistent==0.16.1;python_version<"3.0"', 'pyrsistent;python_version>"3.0"', - 'jsonschema'] + SCHEMA_REGISTRY_REQUIRES + 'jsonschema<4.18.0'] + SCHEMA_REGISTRY_REQUIRES PROTO_REQUIRES = ['protobuf'] + SCHEMA_REGISTRY_REQUIRES diff --git a/src/confluent_kafka/schema_registry/requirements.txt b/src/confluent_kafka/schema_registry/requirements.txt index 83c8d721e..c538fbde0 100644 --- a/src/confluent_kafka/schema_registry/requirements.txt +++ b/src/confluent_kafka/schema_registry/requirements.txt @@ -1,6 +1,6 @@ fastavro>=0.23.0 pyrsistent==0.16.1;python_version<"3.0" pyrsistent;python_version>"3.0" -jsonschema +jsonschema<4.18.0 # Due to https://github.com/tfranzel/drf-spectacular/issues/1132 protobuf requests diff --git a/tests/requirements.txt b/tests/requirements.txt index b5e20efb2..d13eea687 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -10,5 +10,5 @@ fastavro<1.8.0;python_version=="3.7" fastavro>=1.8.4;python_version>"3.7" fastavro avro>=1.11.1,<2 -jsonschema +jsonschema<4.18.0 protobuf diff --git a/tools/build-manylinux.sh b/tools/build-manylinux.sh index 1bbf501ce..70a115f25 100755 --- a/tools/build-manylinux.sh +++ b/tools/build-manylinux.sh @@ -15,7 +15,7 @@ # docker run -t -v $(pwd):/io quay.io/pypa/manylinux2010_x86_64:latest /io/tools/build-manylinux.sh LIBRDKAFKA_VERSION=$1 -PYTHON_VERSIONS=("cp36" "cp37" "cp38" "cp39" "cp310" "cp311" "cp312") +PYTHON_VERSIONS=("cp36" "cp37" "cp38" "cp39" "cp310" "cp311" "cp312" "cp313") if [[ -z "$LIBRDKAFKA_VERSION" ]]; then echo "Usage: $0 " @@ -51,7 +51,7 @@ fi # echo "# Installing basic system dependencies" -yum install -y zlib-devel gcc-c++ python3 curl-devel perl-IPC-Cmd perl-Pod-Html +yum install -y zlib-devel gcc-c++ python3 curl-devel perl-IPC-Cmd perl-Pod-Html libffi-devel echo "# Building librdkafka ${LIBRDKAFKA_VERSION}" $(dirname $0)/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} /usr @@ -92,7 +92,7 @@ for PYBIN in /opt/python/cp*/bin; do "${PYBIN}/pip" -V "${PYBIN}/pip" install --no-index -f /io/wheelhouse confluent_kafka "${PYBIN}/python" -c 'import confluent_kafka; print(confluent_kafka.libversion())' - "${PYBIN}/pip" install -r /io/tests/requirements.txt + "${PYBIN}/pip" install pytest "${PYBIN}/pytest" /io/tests/test_Producer.py echo "## Uninstalling $PYBIN" "${PYBIN}/pip" uninstall -y confluent_kafka diff --git a/tools/mingw-w64/semaphore_commands.sh b/tools/mingw-w64/semaphore_commands.sh index f0a751ac0..cce5ce999 100644 --- a/tools/mingw-w64/semaphore_commands.sh +++ b/tools/mingw-w64/semaphore_commands.sh @@ -8,4 +8,4 @@ export MAKE=mingw32-make # so that Autotools can find it cmd /c mklink /D C:\Python38\python3.exe C:\Python38\python.exe -python -m pip install cibuildwheel==2.16.2 +python -m pip install cibuildwheel==2.21.2 diff --git a/tools/wheels/build-wheels.bat b/tools/wheels/build-wheels.bat index 5e7203407..793daae65 100644 --- a/tools/wheels/build-wheels.bat +++ b/tools/wheels/build-wheels.bat @@ -13,7 +13,7 @@ set WHEELHOUSE=%4 if [%WHEELHOUSE%]==[] goto usage echo on -set CIBW_BUILD=cp36-%BW_ARCH% cp37-%BW_ARCH% cp38-%BW_ARCH% cp39-%BW_ARCH% cp310-%BW_ARCH% cp311-%BW_ARCH% cp312-%BW_ARCH% +set CIBW_BUILD=cp36-%BW_ARCH% cp37-%BW_ARCH% cp38-%BW_ARCH% cp39-%BW_ARCH% cp310-%BW_ARCH% cp311-%BW_ARCH% cp312-%BW_ARCH% cp313-%BW_ARCH% set CIBW_BEFORE_BUILD=python -m pip install delvewheel==1.1.4 set CIBW_TEST_REQUIRES=-r tests/requirements.txt set CIBW_TEST_COMMAND=pytest {project}\tests\test_Producer.py diff --git a/tools/wheels/build-wheels.sh b/tools/wheels/build-wheels.sh index c0f21f882..a785120b9 100755 --- a/tools/wheels/build-wheels.sh +++ b/tools/wheels/build-wheels.sh @@ -10,15 +10,14 @@ this_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Skip PyPy, Python2, old Python3 versions, musl, and x86 builds. export CIBW_SKIP="pp* cp27-* cp35-* *i686 *musllinux* $CIBW_SKIP" # Run a simple test suite -export CIBW_TEST_REQUIRES="-r tests/requirements.txt" +export CIBW_TEST_REQUIRES="pytest" export CIBW_TEST_COMMAND="pytest {project}/tests/test_Producer.py" export CIBW_MANYLINUX_X86_64_IMAGE="manylinux_2_28" export CIBW_MANYLINUX_AARCH64_IMAGE="manylinux_2_28" - librdkafka_version=$1 wheeldir=$2 -cibuildwheel_version="2.16.2" +cibuildwheel_version=${3:-"2.21.2"} if [[ -z $wheeldir ]]; then echo "Usage: $0 " From 7a20188a98110508c386482231fe55cfda1601a3 Mon Sep 17 00:00:00 2001 From: Pranav Rathi <4427674+pranavrth@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:48:28 +0530 Subject: [PATCH 2/3] Updated cibuildwheel to the latest version --- tools/mingw-w64/semaphore_commands.sh | 2 +- tools/wheels/build-wheels.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/mingw-w64/semaphore_commands.sh b/tools/mingw-w64/semaphore_commands.sh index cce5ce999..c6ecd4957 100644 --- a/tools/mingw-w64/semaphore_commands.sh +++ b/tools/mingw-w64/semaphore_commands.sh @@ -8,4 +8,4 @@ export MAKE=mingw32-make # so that Autotools can find it cmd /c mklink /D C:\Python38\python3.exe C:\Python38\python.exe -python -m pip install cibuildwheel==2.21.2 +python -m pip install cibuildwheel==2.21.3 diff --git a/tools/wheels/build-wheels.sh b/tools/wheels/build-wheels.sh index a785120b9..5ccf0b0b1 100755 --- a/tools/wheels/build-wheels.sh +++ b/tools/wheels/build-wheels.sh @@ -17,7 +17,7 @@ export CIBW_MANYLINUX_AARCH64_IMAGE="manylinux_2_28" librdkafka_version=$1 wheeldir=$2 -cibuildwheel_version=${3:-"2.21.2"} +cibuildwheel_version=${3:-"2.21.3"} if [[ -z $wheeldir ]]; then echo "Usage: $0 " From bb467e3112b61a9a90854a1ad545f32067981b59 Mon Sep 17 00:00:00 2001 From: Pranav Rathi <4427674+pranavrth@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:24:12 +0530 Subject: [PATCH 3/3] Removed unnecessary changes --- .semaphore/semaphore.yml | 2 +- examples/requirements.txt | 2 +- setup.py | 2 +- src/confluent_kafka/schema_registry/requirements.txt | 2 +- tests/requirements.txt | 2 +- tools/build-manylinux.sh | 2 +- tools/wheels/build-wheels.bat | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 62b735f65..ffce1b5ba 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -47,7 +47,7 @@ blocks: - name: ARCH value: x64 - name: CIBW_SKIP - value: cp36-* cp37-* cp-* cp38-* cp39-* cp310-* cp311-* cp312-* + value: cp36-* cp37-* cp38-* cp39-* cp310-* cp311-* cp312-* - name: CIBW_ENVIRONMENT_MACOS value: MACOSX_DEPLOYMENT_TARGET=13 jobs: diff --git a/examples/requirements.txt b/examples/requirements.txt index 3f434b94d..5be5932d9 100644 --- a/examples/requirements.txt +++ b/examples/requirements.txt @@ -5,6 +5,6 @@ uvicorn six pyrsistent==0.16.1;python_version<"3.0" pyrsistent;python_version>"3.0" -jsonschema<4.18.0 +jsonschema protobuf requests diff --git a/setup.py b/setup.py index 0b771dc5e..fe12df2d6 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ JSON_REQUIRES = ['pyrsistent==0.16.1;python_version<"3.0"', 'pyrsistent;python_version>"3.0"', - 'jsonschema<4.18.0'] + SCHEMA_REGISTRY_REQUIRES + 'jsonschema'] + SCHEMA_REGISTRY_REQUIRES PROTO_REQUIRES = ['protobuf'] + SCHEMA_REGISTRY_REQUIRES diff --git a/src/confluent_kafka/schema_registry/requirements.txt b/src/confluent_kafka/schema_registry/requirements.txt index c538fbde0..83c8d721e 100644 --- a/src/confluent_kafka/schema_registry/requirements.txt +++ b/src/confluent_kafka/schema_registry/requirements.txt @@ -1,6 +1,6 @@ fastavro>=0.23.0 pyrsistent==0.16.1;python_version<"3.0" pyrsistent;python_version>"3.0" -jsonschema<4.18.0 # Due to https://github.com/tfranzel/drf-spectacular/issues/1132 +jsonschema protobuf requests diff --git a/tests/requirements.txt b/tests/requirements.txt index d13eea687..b5e20efb2 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -10,5 +10,5 @@ fastavro<1.8.0;python_version=="3.7" fastavro>=1.8.4;python_version>"3.7" fastavro avro>=1.11.1,<2 -jsonschema<4.18.0 +jsonschema protobuf diff --git a/tools/build-manylinux.sh b/tools/build-manylinux.sh index 70a115f25..f3a596a73 100755 --- a/tools/build-manylinux.sh +++ b/tools/build-manylinux.sh @@ -51,7 +51,7 @@ fi # echo "# Installing basic system dependencies" -yum install -y zlib-devel gcc-c++ python3 curl-devel perl-IPC-Cmd perl-Pod-Html libffi-devel +yum install -y zlib-devel gcc-c++ python3 curl-devel perl-IPC-Cmd perl-Pod-Html echo "# Building librdkafka ${LIBRDKAFKA_VERSION}" $(dirname $0)/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} /usr diff --git a/tools/wheels/build-wheels.bat b/tools/wheels/build-wheels.bat index 793daae65..d23ef4311 100644 --- a/tools/wheels/build-wheels.bat +++ b/tools/wheels/build-wheels.bat @@ -15,7 +15,7 @@ echo on set CIBW_BUILD=cp36-%BW_ARCH% cp37-%BW_ARCH% cp38-%BW_ARCH% cp39-%BW_ARCH% cp310-%BW_ARCH% cp311-%BW_ARCH% cp312-%BW_ARCH% cp313-%BW_ARCH% set CIBW_BEFORE_BUILD=python -m pip install delvewheel==1.1.4 -set CIBW_TEST_REQUIRES=-r tests/requirements.txt +set CIBW_TEST_REQUIRES=pytest set CIBW_TEST_COMMAND=pytest {project}\tests\test_Producer.py rem set CIBW_BUILD_VERBOSITY=3 set include=%cd%\%DEST%\build\native\include