Merge branch 'main' of https://github.com/apache/airflow into kalyan/…

…AIP-84/list_dag_runs

.github/workflows/basic-tests.yml

@@ -187,27 +187,30 @@ jobs:
         working-directory: ./airflow-client-python
       - name: Install hatch
         run: |
-          python -m pip install --upgrade pipx
-          pipx ensurepath
-          pipx install hatch --force
+          python -m pip install --upgrade uv
+          uv tool install hatch
       - name: Run tests
         run: hatch run run-coverage
         env:
           HATCH_ENV: "test"
         working-directory: ./clients/python
-      - name: "Prepare FAB+standard provider packages: wheel"
-        run: >
-            breeze release-management prepare-provider-packages fab standard \
-              --package-format wheel --skip-tag-check
-      - name: "Install Airflow with fab for webserver tests"
-        run: pip install . dist/apache_airflow_providers_fab-*.whl
-      - name: "Install Airflow with standard provider for webserver tests"
-        run: pip install . dist/apache_airflow_providers_standard-*.whl
-      - name: "Prepare Task SDK package: wheel"
-        run: >
-          breeze release-management prepare-task-sdk-package --package-format wheel
-      - name: "Install Task SDK package"
-        run: pip install ./dist/apache_airflow_task_sdk-*.whl
+      - name: "Install source version of required packages"
+        run: |
+            breeze release-management prepare-provider-packages \
+              fab \
+              standard \
+              common.sql \
+              sqlite \
+              --package-format wheel \
+              --skip-tag-check \
+              --version-suffix-for-pypi dev0
+            pip install . \
+              dist/apache_airflow_providers_fab-*.whl \
+              dist/apache_airflow_providers_standard-*.whl \
+              dist/apache_airflow_providers_common_sql-*.whl \
+              dist/apache_airflow_providers_sqlite-*.whl
+            breeze release-management prepare-task-sdk-package --package-format wheel
+            pip install ./dist/apache_airflow_task_sdk-*.whl
       - name: "Install Python client"
         run: pip install ./dist/apache_airflow_client-*.whl
       - name: "Initialize Airflow DB and start webserver"

.github/workflows/check-providers.yml

@@ -102,8 +102,8 @@ jobs:
         run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
       - name: "Verify wheel packages with twine"
         run: |
-          pipx uninstall twine || true
-          pipx install twine && twine check dist/*.whl
+          uv tool uninstall twine || true
+          uv tool install twine && twine check dist/*.whl
       - name: "Test providers issue generation automatically"
         run: >
           breeze release-management generate-issue-content-providers
@@ -170,8 +170,8 @@ jobs:
           --version-suffix-for-pypi dev0 --package-format sdist
       - name: "Verify sdist packages with twine"
         run: |
-          pipx uninstall twine || true
-          pipx install twine && twine check dist/*.tar.gz
+          uv tool uninstall twine || true
+          uv tool install twine && twine check dist/*.tar.gz
       - name: "Generate source constraints from CI image"
         shell: bash
         run: >

.github/workflows/ci.yml

@@ -127,6 +127,7 @@ jobs:
       latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
       chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
       has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
+      only-new-ui-files: ${{ steps.selective-checks.outputs.only-new-ui-files }}
       source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
       pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
       in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
@@ -279,7 +280,9 @@ jobs:
     name: "Generate constraints"
     needs: [build-info, wait-for-ci-images]
     uses: ./.github/workflows/generate-constraints.yml
-    if: needs.build-info.outputs.ci-image-build == 'true'
+    if: >
+      needs.build-info.outputs.ci-image-build == 'true' &&
+      needs.build-info.outputs.only-new-ui-files != 'true'
     with:
       runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
@@ -381,7 +384,7 @@ jobs:
       run-migration-tests: "true"
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
 
   tests-mysql:
     name: "MySQL tests"
@@ -406,7 +409,7 @@ jobs:
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       run-migration-tests: "true"
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
 
   tests-sqlite:
     name: "Sqlite tests"
@@ -433,7 +436,7 @@ jobs:
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       run-migration-tests: "true"
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
 
   tests-non-db:
     name: "Non-DB tests"
@@ -459,7 +462,7 @@ jobs:
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
 
   tests-special:
     name: "Special tests"
@@ -664,7 +667,8 @@ jobs:
       run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }}
     if: >
       ( needs.build-info.outputs.run-task-sdk-tests == 'true' ||
-      needs.build-info.outputs.run-tests == 'true')
+      needs.build-info.outputs.run-tests == 'true' &&
+      needs.build-info.outputs.only-new-ui-files != 'true')
 
   finalize-tests:
     name: Finalize tests

.github/workflows/news-fragment.yml

@@ -38,7 +38,8 @@ jobs:
 
       - name: Check news fragment
         run: >
-          pipx run towncrier check
+          python -m pip install --upgrade uv &&
+          uv tool run towncrier check
           --dir .
           --config newsfragments/config.toml
           --compare-with origin/${{ github.base_ref }}

.github/workflows/task-sdk-tests.yml

@@ -77,8 +77,8 @@ jobs:
           breeze release-management prepare-task-sdk-package --package-format wheel
       - name: "Verify wheel packages with twine"
         run: |
-          pipx uninstall twine || true
-          pipx install twine && twine check dist/*.whl
+          uv tool uninstall twine || true
+          uv tool install twine && twine check dist/*.whl
       - name: >
           Run unit tests for Airflow Task SDK:Python ${{ matrix.python-version }}
         run: >

.pre-commit-config.yaml

@@ -360,7 +360,7 @@ repos:
         types_or: [python, pyi]
         args: [--fix]
         require_serial: true
-        additional_dependencies: ["ruff==0.7.2"]
+        additional_dependencies: ["ruff==0.7.3"]
         exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py|^performance/tests/test_.*.py
       - id: ruff-format
         name: Run 'ruff format'
@@ -370,7 +370,7 @@ repos:
         types_or: [python, pyi]
         args: []
         require_serial: true
-        additional_dependencies: ["ruff==0.7.2"]
+        additional_dependencies: ["ruff==0.7.3"]
         exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py$
       - id: replace-bad-characters
         name: Replace bad characters
@@ -477,21 +477,21 @@ repos:
         files: ^docs/apache-airflow/extra-packages-ref\.rst$|^hatch_build.py
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/check_extra_packages_ref.py
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.25.0', 'tabulate']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3', 'tabulate']
       - id: check-hatch-build-order
         name: Check order of dependencies in hatch_build.py
         language: python
         files: ^hatch_build.py$
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/check_order_hatch_build.py
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.25.0']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3']
       - id: update-extras
         name: Update extras in documentation
         entry: ./scripts/ci/pre_commit/insert_extras.py
         language: python
         files: ^contributing-docs/12_airflow_dependencies_and_extras.rst$|^INSTALL$|^providers/src/airflow/providers/.*/provider\.yaml$|^Dockerfile.*
         pass_filenames: false
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.25.0']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3']
       - id: check-extras-order
         name: Check order of extras in Dockerfile
         entry: ./scripts/ci/pre_commit/check_order_dockerfile_extras.py
@@ -667,7 +667,8 @@ repos:
           ^.*RELEASE_NOTES\.rst$|
           ^contributing-docs/03_contributors_quick_start.rst$|
           ^.*\.(png|gif|jp[e]?g|tgz|lock)$|
-          git
+          git|
+          ^newsfragments/43368\.significant\.rst$
       - id: check-base-operator-partial-arguments
         name: Check BaseOperator and partial() arguments
         language: python

Dockerfile

@@ -890,7 +890,7 @@ function install_airflow() {
 
         # Similarly we need _a_ file for task_sdk too
         mkdir -p ./task_sdk/src/airflow/sdk/
-        touch ./task_sdk/src/airflow/__init__.py
+        echo '__version__ = "0.0.0dev0"' > ./task_sdk/src/airflow/sdk/__init__.py
 
         trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT
 

Dockerfile.ci

@@ -634,35 +634,6 @@ function common::import_trusted_gpg() {
 }
 EOF
 
-# The content below is automatically copied from scripts/docker/install_pipx_tools.sh
-COPY <<"EOF" /install_pipx_tools.sh
-#!/usr/bin/env bash
-. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
-
-function install_pipx_tools() {
-    echo
-    echo "${COLOR_BLUE}Installing pipx tools${COLOR_RESET}"
-    echo
-    # Make sure PIPX is installed in latest version
-    ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pipx>=1.2.1"
-    if [[ $(uname -m) != "aarch64" ]]; then
-        # Do not install mssql-cli for ARM
-        # Install all the tools we need available in command line but without impacting the current environment
-        pipx install mssql-cli
-
-        # Unfortunately mssql-cli installed by `pipx` does not work out of the box because it uses
-        # its own execution bash script which is not compliant with the auto-activation of
-        # pipx venvs - we need to manually patch Python executable in the script to fix it: ¯\_(ツ)_/¯
-        sed "s/python /\/root\/\.local\/pipx\/venvs\/mssql-cli\/bin\/python /" -i /root/.local/bin/mssql-cli
-    fi
-}
-
-common::get_colors
-common::get_packaging_tool
-
-install_pipx_tools
-EOF
-
 # The content below is automatically copied from scripts/docker/install_airflow.sh
 COPY <<"EOF" /install_airflow.sh
 #!/usr/bin/env bash
@@ -689,7 +660,7 @@ function install_airflow() {
 
         # Similarly we need _a_ file for task_sdk too
         mkdir -p ./task_sdk/src/airflow/sdk/
-        touch ./task_sdk/src/airflow/__init__.py
+        echo '__version__ = "0.0.0dev0"' > ./task_sdk/src/airflow/sdk/__init__.py
 
         trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT
 
@@ -1381,13 +1352,10 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
 # The PATH is needed for PIPX to find the tools installed
 ENV PATH="/root/.local/bin:${PATH}"
 
-COPY --from=scripts install_pipx_tools.sh /scripts/docker/
-
 # Install useful command line tools in their own virtualenv so that they do not clash with
 # dependencies installed in Airflow also reinstall PIP and UV to make sure they are installed
 # in the version specified above
-RUN bash /scripts/docker/install_packaging_tools.sh; \
-    bash /scripts/docker/install_pipx_tools.sh
+RUN bash /scripts/docker/install_packaging_tools.sh
 
 # Airflow sources change frequently but dependency configuration won't change that often
 # We copy pyproject.toml and other files needed to perform setup of dependencies