From 890610f1d9e2496958ff55773bdd7ca8adc5c157 Mon Sep 17 00:00:00 2001
From: Evgenii Kopanev <ekopanev@yandex-team.ru>
Date: Tue, 7 May 2024 10:57:54 +0200
Subject: [PATCH 1/7] Build ch-backup with custom python

---
 Dockerfile-deb-python-build        | 60 ++++++++++++++++++++++++++++++
 Dockerfile-python                  | 38 +++++++++++++++++++
 Dockerfile-python-build            | 41 ++++++++++++++++++++
 Makefile                           | 20 ++++++++--
 build_deb_in_docker.sh             |  2 +-
 build_deb_with_python_in_docker.sh | 44 ++++++++++++++++++++++
 build_python_in_docker.sh          | 26 +++++++++++++
 debian/rules                       |  3 +-
 requirements.txt                   |  7 +++-
 9 files changed, 234 insertions(+), 7 deletions(-)
 create mode 100644 Dockerfile-deb-python-build
 create mode 100644 Dockerfile-python
 create mode 100644 Dockerfile-python-build
 create mode 100755 build_deb_with_python_in_docker.sh
 create mode 100755 build_python_in_docker.sh

diff --git a/Dockerfile-deb-python-build b/Dockerfile-deb-python-build
new file mode 100644
index 00000000..fcdb16bc
--- /dev/null
+++ b/Dockerfile-deb-python-build
@@ -0,0 +1,60 @@
+ARG PYTHON_BUILD_NAME
+ARG BASE_IMAGE=ubuntu:22.04
+
+FROM --platform=$TARGETPLATFORM $PYTHON_BUILD_NAME-build:latest as pybuild
+
+FROM --platform=$TARGETPLATFORM $BASE_IMAGE
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG TARGET_PYTHON_VERSION
+
+# todo: use args
+RUN mkdir -p /opt/yandex/ch-backup/python
+
+COPY --from=pybuild /opt/yandex/ch-backup/python /opt/yandex/ch-backup/python
+
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        # Debian packaging tools
+        build-essential \
+        debhelper \
+        devscripts \
+        fakeroot \
+        # Managing keys for debian package signing
+        gpg \
+        gpg-agent \
+        # Python packaging tools
+        # python3-dev \
+        python3-pip \
+        # python3-setuptools \
+        python3-venv \
+        # Misc
+        curl \
+        locales \
+        # For building PyNacl library
+        libffi-dev libssl-dev libboost-all-dev libsodium-dev \
+    # Configure locales
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8 \
+    # Ensure that `python` refers to `python3` so that poetry works.
+    # It makes sense for ubuntu:18.04
+    && ln -sf /opt/yandex/ch-backup/python/bin/python3.12 /usr/bin/python \
+    && ln -sf /opt/yandex/ch-backup/python/bin/python3.12 /usr/bin/python3
+
+# install extra dependencies
+# RUN /usr/bin/python3 -m pip install setuptools
+
+# COPY --from=pybuild /usr/local/bin /opt/yandex/ch-backup/bin
+
+# Project directory must be mounted here
+VOLUME /src
+WORKDIR /src
+
+# For compiling PyNACL library, which is used by ch-backup
+# See https://pynacl.readthedocs.io/en/latest/install/#linux-source-build
+ENV SODIUM_INSTALL=system
+ENV PYTHON=/opt/yandex/ch-backup/python/bin/python3.12
+ENV TARGET_PYTHON_VERSION=$TARGET_PYTHON_VERSION
+
+CMD ["make", "build-deb-package-local"]
diff --git a/Dockerfile-python b/Dockerfile-python
new file mode 100644
index 00000000..684bd74b
--- /dev/null
+++ b/Dockerfile-python
@@ -0,0 +1,38 @@
+FROM my_python:latest as pybuild
+
+FROM ubuntu:18.04
+
+RUN mkdir -p /opt/yandex/ch-backup
+
+COPY --from=pybuild /usr/local/bin /opt/yandex/ch-backup/bin
+COPY --from=pybuild /usr/local/lib /opt/yandex/ch-backup/lib
+
+WORKDIR /src
+
+CMD ["bash"]
+
+
+# ARG TARGET_PYTHON=3.10
+
+# FROM python:$TARGET_PYTHON as pybuild
+
+# # ARG BASE_IMAGE=ubuntu:22.04
+# # FROM --platform=$TARGETPLATFORM $BASE_IMAGE
+# FROM ubuntu:18.04
+
+# RUN mkdir -p /opt/yandex/ch-backup
+
+# # COPY --from=pybuild /usr/local/bin/python3.12 /opt/yandex/ch-backup/bin/python3.12
+# COPY --from=pybuild /usr/local/bin/python$TARGET_PYTHON /usr/local/bin/python$TARGET_PYTHON
+
+# COPY --from=pybuild /usr/local/lib/ /usr/local/lib/
+
+# # COPY --from=pybuild /usr/local/lib/libpython3.12.so /usr/local/lib/libpython3.12.so
+# # COPY --from=pybuild /usr/local/lib/libpython3.12.so.1.0 /usr/local/lib/libpython3.12.so.1.0
+# # COPY --from=pybuild /usr/local/lib/libpython3.so /usr/local/lib/libpython3.so
+
+# # Project directory must be mounted here
+# VOLUME /src
+# WORKDIR /src
+
+# CMD ["bash"]
diff --git a/Dockerfile-python-build b/Dockerfile-python-build
new file mode 100644
index 00000000..5626edb2
--- /dev/null
+++ b/Dockerfile-python-build
@@ -0,0 +1,41 @@
+ARG BASE_IMAGE=ubuntu:18.04
+FROM --platform=$TARGETPLATFORM $BASE_IMAGE
+
+RUN set -ex && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install \
+        -y --no-install-recommends \
+        build-essential \
+        libbz2-dev \
+        libffi-dev \
+        libgdbm-dev \
+        libncurses5-dev \
+        libnss3-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        libssl-dev \
+        wget \
+        zlib1g-dev \
+    && \
+    apt-get clean
+
+ARG PY_VERSION=3.12.3
+ARG PYTHON_SRC=/src/python
+
+RUN mkdir -p $PYTHON_SRC
+
+RUN set -ex && \
+    wget --no-check-certificate "https://www.python.org/ftp/python/$PY_VERSION/Python-$PY_VERSION.tgz" && \
+    tar -xf Python-$PY_VERSION.tgz && \
+    cd Python-$PY_VERSION/ && \
+    ./configure \
+        --build="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
+        --prefix=$PYTHON_SRC \
+        --enable-optimizations \
+    && \
+    make -j $(nproc) && \
+    make altinstall && \
+    rm /Python-$PY_VERSION.tgz && \
+    rm -rf /Python-$PY_VERSION
+
+WORKDIR /src
diff --git a/Makefile b/Makefile
index c8865d16..03d875ca 100644
--- a/Makefile
+++ b/Makefile
@@ -44,7 +44,6 @@ export DEB_TARGET_PLATFORM ?=
 # If it is not provided, default value in Dockerfile is used
 export DEB_BUILD_DISTRIBUTION ?=
 
-
 .PHONY: build
 build: install-deps ch_backup/version.txt
 
@@ -109,7 +108,7 @@ clean-pycache:
 
 
 .PHONY: install
-install:
+install: .copy-target-python
 	@echo "Installing into $(INSTALL_DIR)"
 	$(PYTHON) -m venv $(INSTALL_DIR)
 	$(INSTALL_DIR)/bin/pip install -r requirements.txt
@@ -189,7 +188,7 @@ format: install-deps
 
 
 ch_backup/version.txt:
-	@echo "2.$$(git rev-list HEAD --count).$$(git rev-parse --short HEAD | perl -ne 'print hex $$_')" > ch_backup/version.txt
+	@echo "0.0.0.8" > ch_backup/version.txt
 
 
 .PHONY: install-deps
@@ -211,6 +210,21 @@ check-environment:
 	touch .install-deps
 
 
+.PHONY: build-python
+build-python:
+	PYTHON_SRC=${INSTALL_DIR}/python ./build_python_in_docker.sh
+
+.PHONY: .copy-target-python
+.copy-target-python:
+	@echo "Target python version: ${TARGET_PYTHON_VERSION}"
+	@if [[ -n "${TARGET_PYTHON_VERSION}" ]]; then \
+		echo "Copying custom python to ${INSTALL_DIR}/python"; \
+		mkdir -p ${INSTALL_DIR}/python; \
+		cp -R /opt/yandex/ch-backup/python ${INSTALL_DIR}; \
+	else \
+		echo "Custom target python version is not set."; \
+	fi
+
 .PHONY: help
 help:
 	@echo "Targets:"
diff --git a/build_deb_in_docker.sh b/build_deb_in_docker.sh
index 790f3e8d..5c9dae91 100755
--- a/build_deb_in_docker.sh
+++ b/build_deb_in_docker.sh
@@ -6,7 +6,7 @@ BUILD_IMAGE=${PROJECT_NAME}-build
 BUILD_ARGS=()
 
 # Compose image name and build arguments
-# Example of image name "clickhouse-tools-build-linux-amd64-linux-bionic"		
+# Example of image name "clickhouse-tools-build-linux-amd64-linux-bionic"
 if [[ -n "${DEB_TARGET_PLATFORM}" ]]; then
     BUILD_ARGS+=(--platform=${DEB_TARGET_PLATFORM})
     BUILD_IMAGE="${BUILD_IMAGE}-${DEB_TARGET_PLATFORM}"
diff --git a/build_deb_with_python_in_docker.sh b/build_deb_with_python_in_docker.sh
new file mode 100755
index 00000000..cc5d7f3d
--- /dev/null
+++ b/build_deb_with_python_in_docker.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -e
+
+BUILD_IMAGE=${PROJECT_NAME}-build
+BUILD_ARGS=()
+
+# Compose image name and build arguments
+# Example of image name "clickhouse-tools-build-linux-amd64-linux-bionic"
+if [[ -n "${DEB_TARGET_PLATFORM}" ]]; then
+    BUILD_ARGS+=(--platform=${DEB_TARGET_PLATFORM})
+    BUILD_IMAGE="${BUILD_IMAGE}-${DEB_TARGET_PLATFORM}"
+fi
+if [[ -n "${DEB_BUILD_DISTRIBUTION}" ]]; then
+    BUILD_ARGS+=(--build-arg BASE_IMAGE=${DEB_BUILD_DISTRIBUTION})
+    BUILD_IMAGE="${BUILD_IMAGE}-${DEB_BUILD_DISTRIBUTION}"
+fi
+if [[ -n "${PYTHON_BUILD_NAME}" ]]; then
+    BUILD_ARGS+=(--build-arg PYTHON_BUILD_NAME=${PYTHON_BUILD_NAME})
+fi
+if [[ -n "${TARGET_PYTHON_VERSION}" ]]; then
+    BUILD_ARGS+=(--build-arg TARGET_PYTHON_VERSION=${TARGET_PYTHON_VERSION})
+fi
+
+# Normalize docker image name
+BUILD_IMAGE=$(echo ${BUILD_IMAGE} | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g')
+
+RUN_ARGS=( \
+    -v ${PWD}:/src \
+    --env BUILD_DEB_OUTPUT_DIR="${BUILD_DEB_OUTPUT_DIR}" \
+    --env DEB_SIGN_KEY="${DEB_SIGN_KEY}" \
+    --env DEB_SIGN_KEY_ID="${DEB_SIGN_KEY_ID}" \
+)
+# Mount signing key file if its path is provided
+if [[ -n "${DEB_SIGN_KEY_PATH}" ]]; then
+    RUN_ARGS+=( \
+        -v ${DEB_SIGN_KEY_PATH}:/signing_key \
+        --env DEB_SIGN_KEY_PATH=/signing_key \
+    )
+fi
+
+echo "Build args: ${BUILD_ARGS[@]}"
+docker build "${BUILD_ARGS[@]}" -t "${BUILD_IMAGE}" -f Dockerfile-deb-python-build .
+docker run "${RUN_ARGS[@]}" "${BUILD_IMAGE}"
diff --git a/build_python_in_docker.sh b/build_python_in_docker.sh
new file mode 100755
index 00000000..3f471adf
--- /dev/null
+++ b/build_python_in_docker.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+BUILD_IMAGE=${PYTHON_BUILD_NAME}-build
+BUILD_ARGS=()
+
+# Normalize docker image name
+BUILD_IMAGE=$(echo ${BUILD_IMAGE} | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g')
+
+if [[ -n "${DEB_TARGET_PLATFORM}" ]]; then
+    BUILD_ARGS+=(--platform=${DEB_TARGET_PLATFORM})
+fi
+if [[ -n "${DEB_BUILD_DISTRIBUTION}" ]]; then
+    BUILD_ARGS+=(--build-arg BASE_IMAGE=${DEB_BUILD_DISTRIBUTION})
+fi
+if [[ -n "${TARGET_PYTHON_VERSION}" ]]; then
+    BUILD_ARGS+=(--build-arg PY_VERSION=${TARGET_PYTHON_VERSION})
+fi
+if [[ -n "${PYTHON_SRC}" ]]; then
+    BUILD_ARGS+=(--build-arg PYTHON_SRC=${PYTHON_SRC})
+fi
+
+docker build "${BUILD_ARGS[@]}" -t "${BUILD_IMAGE}" -f Dockerfile-python-build .
+
+echo "${BUILD_IMAGE} was built."
diff --git a/debian/rules b/debian/rules
index 99250ebf..48a46a98 100755
--- a/debian/rules
+++ b/debian/rules
@@ -11,7 +11,8 @@ PYTHON_TO := $(PYTHON_MAJOR).$(shell echo $$(( $(PYTHON_MINOR) + 1 )))
 # but package for Jammy requires python3.10.
 # All this is due to the fact that we put the entire virtual environment in a deb package
 # and that venv links to the system python
-SUBSTVARS := -Vpython:Depends="python3 (>= $(PYTHON_FROM)), python3 (<< $(PYTHON_TO))"
+# SUBSTVARS := -Vpython:Depends="python3 (>= $(PYTHON_FROM)), python3 (<< $(PYTHON_TO))"
+SUBSTVARS := -Vpython:Depends="python3 (>= 3.6), python3 (<< 3.13)"
 
 %:
 	dh $@
diff --git a/requirements.txt b/requirements.txt
index ba26cb5c..253550fb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,8 @@
 requests>2.19
-boto3<1.19
-botocore<1.22
+# boto3<1.19
+boto3<1.21
+# botocore<1.22
+botocore<1.24
 psutil
 PyYAML<5.4
 PyNaCl==1.2.1
@@ -15,3 +17,4 @@ pypeln==0.4.9
 dataclasses>=0.7,<0.8; python_version <"3.7"  # required for pypeln==0.4.9
 typing_extensions>=3.7.4,<4.0; python_version <"3.8"  # required for pypeln==0.4.9
 loguru
+setuptools; python_version >="3.12"  # instead of python3-setuptools package

From b42ff94a2d762700e48428f04a53852991227cb6 Mon Sep 17 00:00:00 2001
From: Evgenii Kopanev <ekopanev@yandex-team.ru>
Date: Wed, 15 May 2024 18:14:08 +0200
Subject: [PATCH 2/7] Python build image cleanup

---
 Dockerfile-python         | 38 ---------------------------------
 Dockerfile-python-build   | 44 ++++++++++++++++++++-------------------
 Makefile                  |  2 +-
 build_python_in_docker.sh |  6 +++---
 4 files changed, 27 insertions(+), 63 deletions(-)
 delete mode 100644 Dockerfile-python

diff --git a/Dockerfile-python b/Dockerfile-python
deleted file mode 100644
index 684bd74b..00000000
--- a/Dockerfile-python
+++ /dev/null
@@ -1,38 +0,0 @@
-FROM my_python:latest as pybuild
-
-FROM ubuntu:18.04
-
-RUN mkdir -p /opt/yandex/ch-backup
-
-COPY --from=pybuild /usr/local/bin /opt/yandex/ch-backup/bin
-COPY --from=pybuild /usr/local/lib /opt/yandex/ch-backup/lib
-
-WORKDIR /src
-
-CMD ["bash"]
-
-
-# ARG TARGET_PYTHON=3.10
-
-# FROM python:$TARGET_PYTHON as pybuild
-
-# # ARG BASE_IMAGE=ubuntu:22.04
-# # FROM --platform=$TARGETPLATFORM $BASE_IMAGE
-# FROM ubuntu:18.04
-
-# RUN mkdir -p /opt/yandex/ch-backup
-
-# # COPY --from=pybuild /usr/local/bin/python3.12 /opt/yandex/ch-backup/bin/python3.12
-# COPY --from=pybuild /usr/local/bin/python$TARGET_PYTHON /usr/local/bin/python$TARGET_PYTHON
-
-# COPY --from=pybuild /usr/local/lib/ /usr/local/lib/
-
-# # COPY --from=pybuild /usr/local/lib/libpython3.12.so /usr/local/lib/libpython3.12.so
-# # COPY --from=pybuild /usr/local/lib/libpython3.12.so.1.0 /usr/local/lib/libpython3.12.so.1.0
-# # COPY --from=pybuild /usr/local/lib/libpython3.so /usr/local/lib/libpython3.so
-
-# # Project directory must be mounted here
-# VOLUME /src
-# WORKDIR /src
-
-# CMD ["bash"]
diff --git a/Dockerfile-python-build b/Dockerfile-python-build
index 5626edb2..646b94f7 100644
--- a/Dockerfile-python-build
+++ b/Dockerfile-python-build
@@ -1,10 +1,12 @@
 ARG BASE_IMAGE=ubuntu:18.04
 FROM --platform=$TARGETPLATFORM $BASE_IMAGE
 
-RUN set -ex && \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install \
-        -y --no-install-recommends \
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG=C.UTF-8
+
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
         build-essential \
         libbz2-dev \
         libffi-dev \
@@ -15,27 +17,27 @@ RUN set -ex && \
         libsqlite3-dev \
         libssl-dev \
         wget \
+        xz-utils \
         zlib1g-dev \
-    && \
-    apt-get clean
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-ARG PY_VERSION=3.12.3
-ARG PYTHON_SRC=/src/python
+ARG PYTHON_INSTALL_PREFIX=/src/python
+RUN mkdir -p $PYTHON_INSTALL_PREFIX
 
-RUN mkdir -p $PYTHON_SRC
+ARG PYTHON_VERSION=3.12.3
 
-RUN set -ex && \
-    wget --no-check-certificate "https://www.python.org/ftp/python/$PY_VERSION/Python-$PY_VERSION.tgz" && \
-    tar -xf Python-$PY_VERSION.tgz && \
-    cd Python-$PY_VERSION/ && \
-    ./configure \
+RUN set -ex \
+    && wget -O python.tar.xz --no-check-certificate "https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz" \
+    && tar -xf python.tar.xz \
+    && rm python.tar.xz \
+    && cd Python-$PYTHON_VERSION/ \
+    && ./configure \
         --build="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
-        --prefix=$PYTHON_SRC \
+        --prefix=$PYTHON_INSTALL_PREFIX \
         --enable-optimizations \
-    && \
-    make -j $(nproc) && \
-    make altinstall && \
-    rm /Python-$PY_VERSION.tgz && \
-    rm -rf /Python-$PY_VERSION
+    && make -j $(nproc) \
+    && make altinstall \
+    && rm -rf /Python-$PYTHON_VERSION
 
-WORKDIR /src
+WORKDIR $PYTHON_INSTALL_PREFIX
diff --git a/Makefile b/Makefile
index 03d875ca..68b5eaa8 100644
--- a/Makefile
+++ b/Makefile
@@ -212,7 +212,7 @@ check-environment:
 
 .PHONY: build-python
 build-python:
-	PYTHON_SRC=${INSTALL_DIR}/python ./build_python_in_docker.sh
+	PYTHON_INSTALL_PREFIX=${INSTALL_DIR}/python ./build_python_in_docker.sh
 
 .PHONY: .copy-target-python
 .copy-target-python:
diff --git a/build_python_in_docker.sh b/build_python_in_docker.sh
index 3f471adf..f3c323be 100755
--- a/build_python_in_docker.sh
+++ b/build_python_in_docker.sh
@@ -15,10 +15,10 @@ if [[ -n "${DEB_BUILD_DISTRIBUTION}" ]]; then
     BUILD_ARGS+=(--build-arg BASE_IMAGE=${DEB_BUILD_DISTRIBUTION})
 fi
 if [[ -n "${TARGET_PYTHON_VERSION}" ]]; then
-    BUILD_ARGS+=(--build-arg PY_VERSION=${TARGET_PYTHON_VERSION})
+    BUILD_ARGS+=(--build-arg PYTHON_VERSION=${TARGET_PYTHON_VERSION})
 fi
-if [[ -n "${PYTHON_SRC}" ]]; then
-    BUILD_ARGS+=(--build-arg PYTHON_SRC=${PYTHON_SRC})
+if [[ -n "${PYTHON_INSTALL_PREFIX}" ]]; then
+    BUILD_ARGS+=(--build-arg PYTHON_INSTALL_PREFIX=${PYTHON_INSTALL_PREFIX})
 fi
 
 docker build "${BUILD_ARGS[@]}" -t "${BUILD_IMAGE}" -f Dockerfile-python-build .

From c7d436118fe415474e8196c68a737cf0c0b7c04a Mon Sep 17 00:00:00 2001
From: Evgenii Kopanev <ekopanev@yandex-team.ru>
Date: Wed, 15 May 2024 18:40:39 +0200
Subject: [PATCH 3/7] Small cleanup and updated dependencies

---
 Dockerfile-deb-python-build        | 10 +---------
 build_deb_with_python_in_docker.sh |  1 -
 requirements.txt                   |  9 +++++----
 3 files changed, 6 insertions(+), 14 deletions(-)

diff --git a/Dockerfile-deb-python-build b/Dockerfile-deb-python-build
index fcdb16bc..a675764c 100644
--- a/Dockerfile-deb-python-build
+++ b/Dockerfile-deb-python-build
@@ -8,9 +8,7 @@ FROM --platform=$TARGETPLATFORM $BASE_IMAGE
 ARG DEBIAN_FRONTEND=noninteractive
 ARG TARGET_PYTHON_VERSION
 
-# todo: use args
 RUN mkdir -p /opt/yandex/ch-backup/python
-
 COPY --from=pybuild /opt/yandex/ch-backup/python /opt/yandex/ch-backup/python
 
 RUN set -ex \
@@ -25,9 +23,7 @@ RUN set -ex \
         gpg \
         gpg-agent \
         # Python packaging tools
-        # python3-dev \
         python3-pip \
-        # python3-setuptools \
         python3-venv \
         # Misc
         curl \
@@ -42,11 +38,6 @@ RUN set -ex \
     && ln -sf /opt/yandex/ch-backup/python/bin/python3.12 /usr/bin/python \
     && ln -sf /opt/yandex/ch-backup/python/bin/python3.12 /usr/bin/python3
 
-# install extra dependencies
-# RUN /usr/bin/python3 -m pip install setuptools
-
-# COPY --from=pybuild /usr/local/bin /opt/yandex/ch-backup/bin
-
 # Project directory must be mounted here
 VOLUME /src
 WORKDIR /src
@@ -54,6 +45,7 @@ WORKDIR /src
 # For compiling PyNACL library, which is used by ch-backup
 # See https://pynacl.readthedocs.io/en/latest/install/#linux-source-build
 ENV SODIUM_INSTALL=system
+
 ENV PYTHON=/opt/yandex/ch-backup/python/bin/python3.12
 ENV TARGET_PYTHON_VERSION=$TARGET_PYTHON_VERSION
 
diff --git a/build_deb_with_python_in_docker.sh b/build_deb_with_python_in_docker.sh
index cc5d7f3d..8192f237 100755
--- a/build_deb_with_python_in_docker.sh
+++ b/build_deb_with_python_in_docker.sh
@@ -39,6 +39,5 @@ if [[ -n "${DEB_SIGN_KEY_PATH}" ]]; then
     )
 fi
 
-echo "Build args: ${BUILD_ARGS[@]}"
 docker build "${BUILD_ARGS[@]}" -t "${BUILD_IMAGE}" -f Dockerfile-deb-python-build .
 docker run "${RUN_ARGS[@]}" "${BUILD_IMAGE}"
diff --git a/requirements.txt b/requirements.txt
index 253550fb..50b6aa48 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,6 @@
 requests>2.19
-# boto3<1.19
-boto3<1.21
-# botocore<1.22
-botocore<1.24
+boto3<1.19; python_version <"3.12"
+botocore<1.22; python_version <"3.12"
 psutil
 PyYAML<5.4
 PyNaCl==1.2.1
@@ -17,4 +15,7 @@ pypeln==0.4.9
 dataclasses>=0.7,<0.8; python_version <"3.7"  # required for pypeln==0.4.9
 typing_extensions>=3.7.4,<4.0; python_version <"3.8"  # required for pypeln==0.4.9
 loguru
+# for python3.12+
 setuptools; python_version >="3.12"  # instead of python3-setuptools package
+boto3<1.21; python_version >="3.12"
+botocore<1.24; python_version >="3.12"

From ffe3f5a53d04c723a3bf5ab54b2df07721f36eb0 Mon Sep 17 00:00:00 2001
From: Evgenii Kopanev <ekopanev@yandex-team.ru>
Date: Wed, 15 May 2024 19:07:32 +0200
Subject: [PATCH 4/7] Using conditional rules for python dep

---
 Makefile     | 3 ++-
 debian/rules | 8 ++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 68b5eaa8..0f2d6d36 100644
--- a/Makefile
+++ b/Makefile
@@ -188,7 +188,8 @@ format: install-deps
 
 
 ch_backup/version.txt:
-	@echo "0.0.0.8" > ch_backup/version.txt
+	@echo "0.0.0.10" > ch_backup/version.txt
+#	@echo "2.$$(git rev-list HEAD --count).$$(git rev-parse --short HEAD | perl -ne 'print hex $$_')" > ch_backup/version.txt
 
 
 .PHONY: install-deps
diff --git a/debian/rules b/debian/rules
index 48a46a98..fe9a9901 100755
--- a/debian/rules
+++ b/debian/rules
@@ -7,12 +7,16 @@ PYTHON_MINOR := $(shell $(PYTHON) -c 'import sys; print(sys.version_info[1])')
 PYTHON_FROM := $(PYTHON_MAJOR).$(PYTHON_MINOR)
 PYTHON_TO := $(PYTHON_MAJOR).$(shell echo $$(( $(PYTHON_MINOR) + 1 )))
 
+# In case of using built-in python version (which is expected to have at least 3.12)
+# we don't care about this dependency and it can be removed at all later.
+ifeq ($(PYTHON_MINOR),12)
+	PYTHON_FROM="3.6"
+endif
 # Use conditional python3 dependency because package for Bionic requires python3.6,
 # but package for Jammy requires python3.10.
 # All this is due to the fact that we put the entire virtual environment in a deb package
 # and that venv links to the system python
-# SUBSTVARS := -Vpython:Depends="python3 (>= $(PYTHON_FROM)), python3 (<< $(PYTHON_TO))"
-SUBSTVARS := -Vpython:Depends="python3 (>= 3.6), python3 (<< 3.13)"
+SUBSTVARS := -Vpython:Depends="python3 (>= $(PYTHON_FROM)), python3 (<< $(PYTHON_TO))"
 
 %:
 	dh $@

From 068b0bd1e0fa3a3b82dacf815d9ea24c18214a9f Mon Sep 17 00:00:00 2001
From: Evgenii Kopanev <ekopanev@yandex-team.ru>
Date: Thu, 16 May 2024 10:21:13 +0200
Subject: [PATCH 5/7] Reverted to proper ch_backup/version.txt target

---
 Makefile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 0f2d6d36..0bfd30c0 100644
--- a/Makefile
+++ b/Makefile
@@ -188,8 +188,7 @@ format: install-deps
 
 
 ch_backup/version.txt:
-	@echo "0.0.0.10" > ch_backup/version.txt
-#	@echo "2.$$(git rev-list HEAD --count).$$(git rev-parse --short HEAD | perl -ne 'print hex $$_')" > ch_backup/version.txt
+	@echo "2.$$(git rev-list HEAD --count).$$(git rev-parse --short HEAD | perl -ne 'print hex $$_')" > ch_backup/version.txt
 
 
 .PHONY: install-deps

From d667b6d37f0392053b38031daaaf9076211876e2 Mon Sep 17 00:00:00 2001
From: Evgenii Kopanev <ekopanev@yandex-team.ru>
Date: Thu, 16 May 2024 17:42:15 +0200
Subject: [PATCH 6/7] Code review fixes with comments and rules condition

---
 Dockerfile-deb-python-build | 3 +--
 Makefile                    | 1 +
 debian/rules                | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile-deb-python-build b/Dockerfile-deb-python-build
index a675764c..fcbea8f6 100644
--- a/Dockerfile-deb-python-build
+++ b/Dockerfile-deb-python-build
@@ -33,8 +33,7 @@ RUN set -ex \
     # Configure locales
     && locale-gen en_US.UTF-8 \
     && update-locale LANG=en_US.UTF-8 \
-    # Ensure that `python` refers to `python3` so that poetry works.
-    # It makes sense for ubuntu:18.04
+    # Ensure that we use our custom built version of python during building ch-backup package.
     && ln -sf /opt/yandex/ch-backup/python/bin/python3.12 /usr/bin/python \
     && ln -sf /opt/yandex/ch-backup/python/bin/python3.12 /usr/bin/python3
 
diff --git a/Makefile b/Makefile
index 0bfd30c0..d4e46ecf 100644
--- a/Makefile
+++ b/Makefile
@@ -229,6 +229,7 @@ build-python:
 help:
 	@echo "Targets:"
 	@echo "  build (default)            Build project. It installs dependencies and generates version.txt."
+	@echo "  build-python               Build docker image with custom python version."
 	@echo "  all                        Alias for \"build lint test-unit test-integration\"."
 	@echo "  lint                       Run all linter tools. Alias for \"isort black codespell ruff pylint mypy bandit\"."
 	@echo "  test-unit                  Run unit tests."
diff --git a/debian/rules b/debian/rules
index fe9a9901..b0fb3500 100755
--- a/debian/rules
+++ b/debian/rules
@@ -9,7 +9,7 @@ PYTHON_TO := $(PYTHON_MAJOR).$(shell echo $$(( $(PYTHON_MINOR) + 1 )))
 
 # In case of using built-in python version (which is expected to have at least 3.12)
 # we don't care about this dependency and it can be removed at all later.
-ifeq ($(PYTHON_MINOR),12)
+ifeq ($(shell test $(PYTHON_MAJOR) -ge 3 && test $(PYTHON_MINOR) -ge 12; echo $$?),0)
 	PYTHON_FROM="3.6"
 endif
 # Use conditional python3 dependency because package for Bionic requires python3.6,

From 36234519ea6b4d2e02bbd0d1d50bf2e5cacfa2eb Mon Sep 17 00:00:00 2001
From: Evgenii Kopanev <ekopanev@yandex-team.ru>
Date: Thu, 16 May 2024 18:51:17 +0200
Subject: [PATCH 7/7] Added test workflow for buildling python

---
 .github/workflows/workflow.yml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 8736645c..0bc65383 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -82,3 +82,27 @@ jobs:
         name: integration-test-logs-py${{ matrix.python }}-clickhouse-${{ matrix.clickhouse }}
         path: staging/logs/
         if-no-files-found: ignore
+
+  test_build_with_python:
+    name: test build with custom python version
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: ["amd64"]
+        distro: ["bionic"]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Build with custom python
+      shell: bash
+      env:
+        PYTHON_BUILD_NAME: ${{ matrix.distro }}-python-${{ matrix.arch }}
+        TARGET_PYTHON_VERSION: 3.12.3
+        DEB_TARGET_PLATFORM: linux/${{ matrix.arch }}
+        DEB_BUILD_DISTRIBUTION: ubuntu:${{ matrix.distro }}
+        DOCKER_BUILDKIT: 1
+      run: |
+        make build-python