From 076da924696dc7d8814a38f442e9ca3eb3e90634 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 23 Oct 2024 14:59:50 +0200 Subject: [PATCH] Check python version that was used to install pre-commit venvs (#43282) Since we moved to Python 3.9 and started usign 3.9+ only features line functools.cache, having Python 3.8 as default python version will result in virtualenvs created with outdated python. This pre-commit checks what is the global version of python3 and it will complain if it is lower than 3.9. This is done via explicit call of "python3", because this is what pre-commit does. Also global setting is needed because otherwise new virtualenvs created by pre-commit might be created using lower python version. --- .pre-commit-config.yaml | 6 ++ contributing-docs/08_static_code_checks.rst | 2 + .../doc/images/output_static-checks.svg | 2 +- .../doc/images/output_static-checks.txt | 2 +- .../src/airflow_breeze/pre_commit_ids.py | 1 + .../ci/pre_commit/check_min_python_version.py | 68 +++++++++++++++++++ 6 files changed, 79 insertions(+), 2 deletions(-) create mode 100755 scripts/ci/pre_commit/check_min_python_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index caad954c2e6fd..f6112cea4c20b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -168,6 +168,12 @@ repos: \.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$ - repo: local hooks: + - id: check-min-python-version + name: Check minimum Python version + entry: ./scripts/ci/pre_commit/check_min_python_version.py + language: python + additional_dependencies: ['rich>=12.4.4'] + require_serial: true - id: update-common-sql-api-stubs name: Check and update common.sql API stubs entry: ./scripts/ci/pre_commit/update_common_sql_api_stubs.py diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index aa9955da1afdd..fc0d4280b9d0e 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -194,6 +194,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------+---------+ | check-merge-conflict | Check that merge conflicts are not being committed | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ +| check-min-python-version | Check minimum Python version | | ++-----------------------------------------------------------+--------------------------------------------------------+---------+ | check-newsfragments-are-valid | Check newsfragments are valid | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ | check-no-airflow-deprecation-in-providers | Do not use DeprecationWarning in providers | | diff --git a/dev/breeze/doc/images/output_static-checks.svg b/dev/breeze/doc/images/output_static-checks.svg index 96a324e22c427..c18571e592e9a 100644 --- a/dev/breeze/doc/images/output_static-checks.svg +++ b/dev/breeze/doc/images/output_static-checks.svg @@ -345,7 +345,7 @@ │check-hatch-build-order | check-hooks-apply | check-incorrect-use-of-LoggingMixin│ │| check-init-decorator-arguments | check-integrations-list-consistent |          │ │check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ -│check-merge-conflict | check-newsfragments-are-valid |                           │ +│check-merge-conflict | check-min-python-version | check-newsfragments-are-valid |│ │check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ │check-only-new-session-with-provide-session |                                    │ │check-persist-credentials-disabled-in-github-workflows |                         │ diff --git a/dev/breeze/doc/images/output_static-checks.txt b/dev/breeze/doc/images/output_static-checks.txt index b0c56ad6640b1..53c5351a1c504 100644 --- a/dev/breeze/doc/images/output_static-checks.txt +++ b/dev/breeze/doc/images/output_static-checks.txt @@ -1 +1 @@ -b4becd0ef113ac04210350ea8f9f98b9 +53b7f32a93cb7dec849138d404c47f6c diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 91b3ad06330ac..9e46069eab306 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -62,6 +62,7 @@ "check-lazy-logging", "check-links-to-example-dags-do-not-use-hardcoded-versions", "check-merge-conflict", + "check-min-python-version", "check-newsfragments-are-valid", "check-no-airflow-deprecation-in-providers", "check-no-providers-in-core-examples", diff --git a/scripts/ci/pre_commit/check_min_python_version.py b/scripts/ci/pre_commit/check_min_python_version.py new file mode 100755 index 0000000000000..825b899241816 --- /dev/null +++ b/scripts/ci/pre_commit/check_min_python_version.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.resolve())) + +from common_precommit_utils import console + +# update this version when we switch to a newer version of Python +required_version = tuple(map(int, "3.9".split("."))) +required_version_str = f"{required_version[0]}.{required_version[1]}" +global_version = tuple( + map( + int, + subprocess.run( + [ + "python3", + "-c", + 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")', + ], + capture_output=True, + text=True, + check=True, + ) + .stdout.strip() + .split("."), + ) +) + + +if global_version < required_version: + console.print(f"[red]Python {required_version_str} or higher is required to install pre-commit.\n") + console.print(f"[green]Current version is {global_version}\n") + console.print( + "[bright_yellow]Please follow those steps:[/]\n\n" + f" * make sure that `python3 --version` is at least {required_version_str}\n" + f" * run `pre-commit clean`\n" + f" * run `pre-commit install --install-hooks`\n\n" + ) + console.print( + "There are various ways you can set `python3` to point to a newer version of Python.\n\n" + f"For example run `pyenv global {required_version_str}` if you use pyenv, or\n" + f"you can use venv with python {required_version_str} when you use pre-commit, or\n" + "you can use `update-alternatives` if you use Ubuntu, or\n" + "you can set `PATH` to point to the newer version of Python.\n\n" + ) + sys.exit(1) +else: + console.print(f"Python version is sufficient: {required_version_str}")