Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/apache/airflow into kalyan/…
Browse files Browse the repository at this point in the history
…AIP-84/list_dag_runs
  • Loading branch information
rawwar committed Nov 15, 2024
2 parents b1ba50f + 19303ca commit 7fa24a2
Showing 636 changed files with 19,175 additions and 6,602 deletions.
35 changes: 19 additions & 16 deletions .github/workflows/basic-tests.yml
Original file line number Diff line number Diff line change
@@ -187,27 +187,30 @@ jobs:
working-directory: ./airflow-client-python
- name: Install hatch
run: |
python -m pip install --upgrade pipx
pipx ensurepath
pipx install hatch --force
python -m pip install --upgrade uv
uv tool install hatch
- name: Run tests
run: hatch run run-coverage
env:
HATCH_ENV: "test"
working-directory: ./clients/python
- name: "Prepare FAB+standard provider packages: wheel"
run: >
breeze release-management prepare-provider-packages fab standard \
--package-format wheel --skip-tag-check
- name: "Install Airflow with fab for webserver tests"
run: pip install . dist/apache_airflow_providers_fab-*.whl
- name: "Install Airflow with standard provider for webserver tests"
run: pip install . dist/apache_airflow_providers_standard-*.whl
- name: "Prepare Task SDK package: wheel"
run: >
breeze release-management prepare-task-sdk-package --package-format wheel
- name: "Install Task SDK package"
run: pip install ./dist/apache_airflow_task_sdk-*.whl
- name: "Install source version of required packages"
run: |
breeze release-management prepare-provider-packages \
fab \
standard \
common.sql \
sqlite \
--package-format wheel \
--skip-tag-check \
--version-suffix-for-pypi dev0
pip install . \
dist/apache_airflow_providers_fab-*.whl \
dist/apache_airflow_providers_standard-*.whl \
dist/apache_airflow_providers_common_sql-*.whl \
dist/apache_airflow_providers_sqlite-*.whl
breeze release-management prepare-task-sdk-package --package-format wheel
pip install ./dist/apache_airflow_task_sdk-*.whl
- name: "Install Python client"
run: pip install ./dist/apache_airflow_client-*.whl
- name: "Initialize Airflow DB and start webserver"
8 changes: 4 additions & 4 deletions .github/workflows/check-providers.yml
Original file line number Diff line number Diff line change
@@ -102,8 +102,8 @@ jobs:
run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
- name: "Verify wheel packages with twine"
run: |
pipx uninstall twine || true
pipx install twine && twine check dist/*.whl
uv tool uninstall twine || true
uv tool install twine && twine check dist/*.whl
- name: "Test providers issue generation automatically"
run: >
breeze release-management generate-issue-content-providers
@@ -170,8 +170,8 @@ jobs:
--version-suffix-for-pypi dev0 --package-format sdist
- name: "Verify sdist packages with twine"
run: |
pipx uninstall twine || true
pipx install twine && twine check dist/*.tar.gz
uv tool uninstall twine || true
uv tool install twine && twine check dist/*.tar.gz
- name: "Generate source constraints from CI image"
shell: bash
run: >
16 changes: 10 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -127,6 +127,7 @@ jobs:
latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
only-new-ui-files: ${{ steps.selective-checks.outputs.only-new-ui-files }}
source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
@@ -279,7 +280,9 @@ jobs:
name: "Generate constraints"
needs: [build-info, wait-for-ci-images]
uses: ./.github/workflows/generate-constraints.yml
if: needs.build-info.outputs.ci-image-build == 'true'
if: >
needs.build-info.outputs.ci-image-build == 'true' &&
needs.build-info.outputs.only-new-ui-files != 'true'
with:
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
@@ -381,7 +384,7 @@ jobs:
run-migration-tests: "true"
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'

tests-mysql:
name: "MySQL tests"
@@ -406,7 +409,7 @@ jobs:
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
run-migration-tests: "true"
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'

tests-sqlite:
name: "Sqlite tests"
@@ -433,7 +436,7 @@ jobs:
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
run-migration-tests: "true"
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'

tests-non-db:
name: "Non-DB tests"
@@ -459,7 +462,7 @@ jobs:
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'

tests-special:
name: "Special tests"
@@ -664,7 +667,8 @@ jobs:
run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }}
if: >
( needs.build-info.outputs.run-task-sdk-tests == 'true' ||
needs.build-info.outputs.run-tests == 'true')
needs.build-info.outputs.run-tests == 'true' &&
needs.build-info.outputs.only-new-ui-files != 'true')
finalize-tests:
name: Finalize tests
3 changes: 2 additions & 1 deletion .github/workflows/news-fragment.yml
Original file line number Diff line number Diff line change
@@ -38,7 +38,8 @@ jobs:

- name: Check news fragment
run: >
pipx run towncrier check
python -m pip install --upgrade uv &&
uv tool run towncrier check
--dir .
--config newsfragments/config.toml
--compare-with origin/${{ github.base_ref }}
4 changes: 2 additions & 2 deletions .github/workflows/task-sdk-tests.yml
Original file line number Diff line number Diff line change
@@ -77,8 +77,8 @@ jobs:
breeze release-management prepare-task-sdk-package --package-format wheel
- name: "Verify wheel packages with twine"
run: |
pipx uninstall twine || true
pipx install twine && twine check dist/*.whl
uv tool uninstall twine || true
uv tool install twine && twine check dist/*.whl
- name: >
Run unit tests for Airflow Task SDK:Python ${{ matrix.python-version }}
run: >
13 changes: 7 additions & 6 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -360,7 +360,7 @@ repos:
types_or: [python, pyi]
args: [--fix]
require_serial: true
additional_dependencies: ["ruff==0.7.2"]
additional_dependencies: ["ruff==0.7.3"]
exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py|^performance/tests/test_.*.py
- id: ruff-format
name: Run 'ruff format'
@@ -370,7 +370,7 @@ repos:
types_or: [python, pyi]
args: []
require_serial: true
additional_dependencies: ["ruff==0.7.2"]
additional_dependencies: ["ruff==0.7.3"]
exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py$
- id: replace-bad-characters
name: Replace bad characters
@@ -477,21 +477,21 @@ repos:
files: ^docs/apache-airflow/extra-packages-ref\.rst$|^hatch_build.py
pass_filenames: false
entry: ./scripts/ci/pre_commit/check_extra_packages_ref.py
additional_dependencies: ['rich>=12.4.4', 'hatchling==1.25.0', 'tabulate']
additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3', 'tabulate']
- id: check-hatch-build-order
name: Check order of dependencies in hatch_build.py
language: python
files: ^hatch_build.py$
pass_filenames: false
entry: ./scripts/ci/pre_commit/check_order_hatch_build.py
additional_dependencies: ['rich>=12.4.4', 'hatchling==1.25.0']
additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3']
- id: update-extras
name: Update extras in documentation
entry: ./scripts/ci/pre_commit/insert_extras.py
language: python
files: ^contributing-docs/12_airflow_dependencies_and_extras.rst$|^INSTALL$|^providers/src/airflow/providers/.*/provider\.yaml$|^Dockerfile.*
pass_filenames: false
additional_dependencies: ['rich>=12.4.4', 'hatchling==1.25.0']
additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3']
- id: check-extras-order
name: Check order of extras in Dockerfile
entry: ./scripts/ci/pre_commit/check_order_dockerfile_extras.py
@@ -667,7 +667,8 @@ repos:
^.*RELEASE_NOTES\.rst$|
^contributing-docs/03_contributors_quick_start.rst$|
^.*\.(png|gif|jp[e]?g|tgz|lock)$|
git
git|
^newsfragments/43368\.significant\.rst$
- id: check-base-operator-partial-arguments
name: Check BaseOperator and partial() arguments
language: python
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -890,7 +890,7 @@ function install_airflow() {

# Similarly we need _a_ file for task_sdk too
mkdir -p ./task_sdk/src/airflow/sdk/
touch ./task_sdk/src/airflow/__init__.py
echo '__version__ = "0.0.0dev0"' > ./task_sdk/src/airflow/sdk/__init__.py

trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT

36 changes: 2 additions & 34 deletions Dockerfile.ci
Original file line number Diff line number Diff line change
@@ -634,35 +634,6 @@ function common::import_trusted_gpg() {
}
EOF

# The content below is automatically copied from scripts/docker/install_pipx_tools.sh
COPY <<"EOF" /install_pipx_tools.sh
#!/usr/bin/env bash
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

function install_pipx_tools() {
echo
echo "${COLOR_BLUE}Installing pipx tools${COLOR_RESET}"
echo
# Make sure PIPX is installed in latest version
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pipx>=1.2.1"
if [[ $(uname -m) != "aarch64" ]]; then
# Do not install mssql-cli for ARM
# Install all the tools we need available in command line but without impacting the current environment
pipx install mssql-cli

# Unfortunately mssql-cli installed by `pipx` does not work out of the box because it uses
# its own execution bash script which is not compliant with the auto-activation of
# pipx venvs - we need to manually patch Python executable in the script to fix it: ¯\_(ツ)_/¯
sed "s/python /\/root\/\.local\/pipx\/venvs\/mssql-cli\/bin\/python /" -i /root/.local/bin/mssql-cli
fi
}

common::get_colors
common::get_packaging_tool

install_pipx_tools
EOF

# The content below is automatically copied from scripts/docker/install_airflow.sh
COPY <<"EOF" /install_airflow.sh
#!/usr/bin/env bash
@@ -689,7 +660,7 @@ function install_airflow() {

# Similarly we need _a_ file for task_sdk too
mkdir -p ./task_sdk/src/airflow/sdk/
touch ./task_sdk/src/airflow/__init__.py
echo '__version__ = "0.0.0dev0"' > ./task_sdk/src/airflow/sdk/__init__.py

trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT

@@ -1381,13 +1352,10 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
# The PATH is needed for PIPX to find the tools installed
ENV PATH="/root/.local/bin:${PATH}"

COPY --from=scripts install_pipx_tools.sh /scripts/docker/

# Install useful command line tools in their own virtualenv so that they do not clash with
# dependencies installed in Airflow also reinstall PIP and UV to make sure they are installed
# in the version specified above
RUN bash /scripts/docker/install_packaging_tools.sh; \
bash /scripts/docker/install_pipx_tools.sh
RUN bash /scripts/docker/install_packaging_tools.sh

# Airflow sources change frequently but dependency configuration won't change that often
# We copy pyproject.toml and other files needed to perform setup of dependencies
Loading

0 comments on commit 7fa24a2

Please sign in to comment.