Support of multiple Python versions in integration tests (#31)

* Support of multiple Python versions in integration tests [MDB-24958] * Graceful worker pool shutdown * Temporary disable "Backup & Restore with long file names" test scenario [MDB-24958] * Set default TAR format as GNU --------- Co-authored-by: Aleksei Filatov <[email protected]>
yandex · Sep 5, 2023 · 2656373 · 2656373
1 parent b3c11d8
commit 2656373
Show file tree

Hide file tree

Showing 8 changed files with 36 additions and 20 deletions.
diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
@@ -36,35 +36,46 @@ jobs:
       run: make bandit
 
   test:
+    name: test (Python ${{ matrix.python }}, ClickHouse ${{ matrix.clickhouse }})
     needs: lint
     strategy:
       fail-fast: false
       matrix:
-        target:
+        include:
         - python: "3.6"
-          ubuntu: "20.04"
+          clickhouse: "latest"
+        - python: "3.7"
+          clickhouse: "latest"
+        - python: "3.8"
+          clickhouse: "latest"
+        - python: "3.9"
+          clickhouse: "latest"
         - python: "3.10"
-          ubuntu: "latest"
-        clickhouse:
-        - "21.8.15.7"
-        - "22.3.20.29"
-        - "22.8.21.38"
-        - "23.3.11.5"
-        - "latest"
-    runs-on: ubuntu-${{ matrix.target.ubuntu }}
+          clickhouse: "21.8.15.7"
+        - python: "3.10"
+          clickhouse: "22.3.20.29"
+        - python: "3.10"
+          clickhouse: "22.8.21.38"
+        - python: "3.10"
+          clickhouse: "23.3.11.5"
+        - python: "3.10"
+          clickhouse: "latest"
+        - python: "3.11"
+          clickhouse: "latest"
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v3
-    - name: set up python ${{ matrix.target.python }}
+    - name: set up python ${{ matrix.python }}
       uses: actions/setup-python@v4
       with:
-        python-version: ${{ matrix.target.python }}
+        python-version: ${{ matrix.python }}
     - name: run unit tests
       run: make test-unit
     - name: run integration tests
       run: CLICKHOUSE_VERSION=${{ matrix.clickhouse }} make test-integration
     - uses: actions/upload-artifact@v3
       if: ${{ failure() }}
       with:
-        name: integration-test-logs-py${{ matrix.target.python }}-clickhouse-${{ matrix.clickhouse }}
+        name: integration-test-logs-py${{ matrix.python }}-clickhouse-${{ matrix.clickhouse }}
         path: staging/logs/
         if-no-files-found: ignore
diff --git a/Makefile b/Makefile
@@ -5,14 +5,16 @@ export COMPOSE_HTTP_TIMEOUT?=300
 export CLICKHOUSE_VERSION?=latest
 
 ifndef NO_VENV
-  export PATH := venv/bin:${PATH}
+  PATH:=venv/bin:${PATH}
 endif
 
+PYTHON_VERSION=$(shell python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
 SESSION_FILE=.session_conf.sav
 INSTALL_DIR=$(DESTDIR)/opt/yandex/ch-backup
 
 TEST_ENV=env \
     PATH=${PATH} \
+    PYTHON_VERSION=${PYTHON_VERSION} \
     PYTHONIOENCODING=${PYTHONIOENCODING} \
     CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION} \
     COMPOSE_HTTP_TIMEOUT=${COMPOSE_HTTP_TIMEOUT}

diff --git a/ch_backup/storage/async_pipeline/base_pipeline/exec_pool.py b/ch_backup/storage/async_pipeline/base_pipeline/exec_pool.py
@@ -64,4 +64,4 @@ def __del__(self) -> None:
         """
         Shutdown pool explicitly to prevent the program from hanging in case of ungraceful termination.
         """
-        self._pool.shutdown(wait=False)
+        self.shutdown()
diff --git a/ch_backup/storage/async_pipeline/stages/filesystem/read_files_tarball_stage.py b/ch_backup/storage/async_pipeline/stages/filesystem/read_files_tarball_stage.py
@@ -55,4 +55,4 @@ def make_tar_header(name: str, file_path: Path) -> bytes:
         tarinfo = tarfile.TarInfo(name)
         stat = file_path.stat()
         tarinfo.mtime, tarinfo.size = int(stat.st_mtime), stat.st_size
-        return tarinfo.tobuf()
+        return tarinfo.tobuf(format=tarfile.GNU_FORMAT)
diff --git a/images/clickhouse/Dockerfile b/images/clickhouse/Dockerfile
@@ -1,10 +1,12 @@
-FROM ubuntu:bionic
+ARG PYTHON_VERSION=3
 
-ENV TZ=Europe/Moscow
-ENV CH_TMP_DIR /var/tmp/ch-backup
+FROM python:${PYTHON_VERSION}-bullseye
 
 ARG CLICKHOUSE_VERSION=latest
 
+ENV TZ=Europe/Moscow
+ENV CH_TMP_DIR=/var/tmp/ch-backup
+
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \
     apt-get update -qq && \
     apt-get upgrade -y && \

diff --git a/setup.py b/setup.py
@@ -48,6 +48,7 @@
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Database",
         "Typing :: Typed",
     ],

diff --git a/tests/integration/configuration.py b/tests/integration/configuration.py
@@ -72,6 +72,7 @@ def create():
                 "depends_on": ["minio", "proxy", "proxy-api", "zookeeper"],
                 "external_links": [f'{s3["host"]}:minio', f'{zk["uri"]}:zookeeper'],
                 "args": {
+                    "PYTHON_VERSION": "$PYTHON_VERSION",
                     "CLICKHOUSE_VERSION": "$CLICKHOUSE_VERSION",
                 },
             },

diff --git a/tests/integration/features/backup_restore.feature b/tests/integration/features/backup_restore.feature
@@ -102,7 +102,6 @@ Feature: Backup & Restore
     When we create clickhouse01 clickhouse backup
     And we restore clickhouse backup #0 to clickhouse02
     Then we got same clickhouse data at clickhouse01 clickhouse02
-# TODO: check deduplication with overdue backups
 
   Scenario: Backup & Restore with non-ascii file names
     When we drop all databases at clickhouse01