diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 657b54af..90459776 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -36,28 +36,39 @@ jobs: run: make bandit test: + name: test (Python ${{ matrix.python }}, ClickHouse ${{ matrix.clickhouse }}) needs: lint strategy: fail-fast: false matrix: - target: + include: - python: "3.6" - ubuntu: "20.04" + clickhouse: "latest" + - python: "3.7" + clickhouse: "latest" + - python: "3.8" + clickhouse: "latest" + - python: "3.9" + clickhouse: "latest" - python: "3.10" - ubuntu: "latest" - clickhouse: - - "21.8.15.7" - - "22.3.20.29" - - "22.8.21.38" - - "23.3.11.5" - - "latest" - runs-on: ubuntu-${{ matrix.target.ubuntu }} + clickhouse: "21.8.15.7" + - python: "3.10" + clickhouse: "22.3.20.29" + - python: "3.10" + clickhouse: "22.8.21.38" + - python: "3.10" + clickhouse: "23.3.11.5" + - python: "3.10" + clickhouse: "latest" + - python: "3.11" + clickhouse: "latest" + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - - name: set up python ${{ matrix.target.python }} + - name: set up python ${{ matrix.python }} uses: actions/setup-python@v4 with: - python-version: ${{ matrix.target.python }} + python-version: ${{ matrix.python }} - name: run unit tests run: make test-unit - name: run integration tests @@ -65,6 +76,6 @@ jobs: - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: - name: integration-test-logs-py${{ matrix.target.python }}-clickhouse-${{ matrix.clickhouse }} + name: integration-test-logs-py${{ matrix.python }}-clickhouse-${{ matrix.clickhouse }} path: staging/logs/ if-no-files-found: ignore diff --git a/Makefile b/Makefile index 0f4eea2e..2bf4dcfd 100644 --- a/Makefile +++ b/Makefile @@ -5,14 +5,16 @@ export COMPOSE_HTTP_TIMEOUT?=300 export CLICKHOUSE_VERSION?=latest ifndef NO_VENV - export PATH := venv/bin:${PATH} + PATH:=venv/bin:${PATH} endif +PYTHON_VERSION=$(shell python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') SESSION_FILE=.session_conf.sav INSTALL_DIR=$(DESTDIR)/opt/yandex/ch-backup TEST_ENV=env \ PATH=${PATH} \ + PYTHON_VERSION=${PYTHON_VERSION} \ PYTHONIOENCODING=${PYTHONIOENCODING} \ CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION} \ COMPOSE_HTTP_TIMEOUT=${COMPOSE_HTTP_TIMEOUT} diff --git a/ch_backup/storage/async_pipeline/base_pipeline/exec_pool.py b/ch_backup/storage/async_pipeline/base_pipeline/exec_pool.py index d0f1b40c..d189f141 100644 --- a/ch_backup/storage/async_pipeline/base_pipeline/exec_pool.py +++ b/ch_backup/storage/async_pipeline/base_pipeline/exec_pool.py @@ -64,4 +64,4 @@ def __del__(self) -> None: """ Shutdown pool explicitly to prevent the program from hanging in case of ungraceful termination. """ - self._pool.shutdown(wait=False) + self.shutdown() diff --git a/ch_backup/storage/async_pipeline/stages/filesystem/read_files_tarball_stage.py b/ch_backup/storage/async_pipeline/stages/filesystem/read_files_tarball_stage.py index 60726d58..8dd0ba01 100644 --- a/ch_backup/storage/async_pipeline/stages/filesystem/read_files_tarball_stage.py +++ b/ch_backup/storage/async_pipeline/stages/filesystem/read_files_tarball_stage.py @@ -55,4 +55,4 @@ def make_tar_header(name: str, file_path: Path) -> bytes: tarinfo = tarfile.TarInfo(name) stat = file_path.stat() tarinfo.mtime, tarinfo.size = int(stat.st_mtime), stat.st_size - return tarinfo.tobuf() + return tarinfo.tobuf(format=tarfile.GNU_FORMAT) diff --git a/images/clickhouse/Dockerfile b/images/clickhouse/Dockerfile index 95e704b6..9b887b43 100644 --- a/images/clickhouse/Dockerfile +++ b/images/clickhouse/Dockerfile @@ -1,10 +1,12 @@ -FROM ubuntu:bionic +ARG PYTHON_VERSION=3 -ENV TZ=Europe/Moscow -ENV CH_TMP_DIR /var/tmp/ch-backup +FROM python:${PYTHON_VERSION}-bullseye ARG CLICKHOUSE_VERSION=latest +ENV TZ=Europe/Moscow +ENV CH_TMP_DIR=/var/tmp/ch-backup + RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ apt-get update -qq && \ apt-get upgrade -y && \ diff --git a/setup.py b/setup.py index 616b197d..9545f19b 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Database", "Typing :: Typed", ], diff --git a/tests/integration/configuration.py b/tests/integration/configuration.py index 10025e97..d596f765 100644 --- a/tests/integration/configuration.py +++ b/tests/integration/configuration.py @@ -72,6 +72,7 @@ def create(): "depends_on": ["minio", "proxy", "proxy-api", "zookeeper"], "external_links": [f'{s3["host"]}:minio', f'{zk["uri"]}:zookeeper'], "args": { + "PYTHON_VERSION": "$PYTHON_VERSION", "CLICKHOUSE_VERSION": "$CLICKHOUSE_VERSION", }, }, diff --git a/tests/integration/features/backup_restore.feature b/tests/integration/features/backup_restore.feature index 927c8250..ff73dfc5 100644 --- a/tests/integration/features/backup_restore.feature +++ b/tests/integration/features/backup_restore.feature @@ -102,7 +102,6 @@ Feature: Backup & Restore When we create clickhouse01 clickhouse backup And we restore clickhouse backup #0 to clickhouse02 Then we got same clickhouse data at clickhouse01 clickhouse02 -# TODO: check deduplication with overdue backups Scenario: Backup & Restore with non-ascii file names When we drop all databases at clickhouse01