Airflow, Spark, OpenTelemetry #143
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: execute whirl ci for examples | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| jobs: | |
| shellcheck: # Job that list subdirectories of ./examples | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: run shellcheck | |
| uses: sudo-bot/action-shellcheck@latest | |
| with: | |
| # https://github.com/koalaman/shellcheck#how-to-use | |
| cli-args: "-x whirl" | |
| directories: # Job that list subdirectories of ./examples | |
| runs-on: ubuntu-latest | |
| outputs: | |
| # generate output name dir by using inner step output | |
| dir: ${{ steps.setdirs.outputs.dir }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - id: setdirs # Give it an id to handle to get step outputs in the outputs key above | |
| # run: echo "::set-output name=dir::$(ls -d ./examples/* | jq -R -s -c 'split("\n")[:-1]')" | |
| run: echo "dir=$(ls -d ./examples/* | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT | |
| # Define step output named dir based on ls command transformed to JSON thanks to jq | |
| examples: # Job that list subdirectories of ./examples | |
| runs-on: ubuntu-latest | |
| outputs: | |
| # generate output name example by using inner step output | |
| example: ${{ steps.setexamples.outputs.example }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - id: setexamples # Give it an id to handle to get step outputs in the outputs key above | |
| run: echo "example=$(ls -d ./examples/* | sed -r 's/\.\/examples\/(.*)/\1/g' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT | |
| # Define step output named example based on ls command transformed to JSON thanks to jq | |
| whirl-ci-default-envs: | |
| needs: [directories] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| example_dir: ${{ fromJson(needs.directories.outputs.dir) }} | |
| python_version: ["3.9", "3.12"] | |
| airflow_version: ["2.2.5", "2.9.1"] | |
| exclude: | |
| - airflow_version: "2.2.5" | |
| python_version: "3.12" | |
| # Needs more memory than available on the runner | |
| - example_dir: ./examples/dbt-spark-example | |
| - example_dir: ./examples/spark-delta-sharing | |
| - example_dir: ./examples/spark-s3-to-hive | |
| # No default dag available for CI | |
| - example_dir: ./examples/airflow-datasets | |
| # Run without parallelism separately | |
| - example_dir: ./examples/dbt-example | |
| # Requires artifact downloaded from github.com/godatadriven/spot, which doesn't publish artifacts yet (2024-12-11) | |
| - example: spark-opentelemetry | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| AIRFLOW_VERSION: ${{ matrix.airflow_version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI ${{ matrix.example_dir }} | |
| working-directory: ${{ matrix.example_dir }} | |
| run: | | |
| echo Run Ci from example directory ${{ matrix.example_dir }} | |
| ../../whirl ci | |
| whirl-ci-default-envs-from-root-dir: | |
| needs: [examples] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| example: ${{ fromJson(needs.examples.outputs.example) }} | |
| python_version: ["3.9", "3.12"] | |
| airflow_version: ["2.2.5", "2.9.1"] | |
| exclude: | |
| - airflow_version: "2.2.5" | |
| python_version: "3.12" | |
| # Needs more memory than available on the runner | |
| - example: dbt-spark-example | |
| - example: spark-delta-sharing | |
| - example: spark-s3-to-hive | |
| # No default dag available for CI | |
| - example: airflow-datasets | |
| # Run without parallelism separately | |
| - example: dbt-example | |
| # Requires artifact downloaded from github.com/godatadriven/spot, which doesn't publish artifacts yet (2024-12-11) | |
| - example: spark-opentelemetry | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| AIRFLOW_VERSION: ${{ matrix.airflow_version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI example ${{ matrix.example }} | |
| run: | | |
| echo Run Ci for example ${{ matrix.example }} | |
| ./whirl -x ${{ matrix.example }} ci | |
| whirl-ci-dbt-example: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| python_version: ["3.9", "3.12"] | |
| airflow_version: ["2.2.5", "2.9.1"] | |
| exclude: | |
| - airflow_version: "2.2.5" | |
| python_version: "3.12" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| AIRFLOW_VERSION: ${{ matrix.airflow_version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI dbt example | |
| working-directory: ./examples/dbt-example | |
| run: | | |
| echo Run Ci from dbt-example directory | |
| ../../whirl ci | |
| whirl-ci-opentelemetry-example: | |
| if: false # Disabled until github.com/godatadriven/spot has public Releases (need a stable download URL) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI opentelemetry example | |
| working-directory: ./examples/spark-opentelemetry/ | |
| run: | | |
| curl -fLSs \ | |
| -o ./whirl.setup.d/spot-complete.jar \ | |
| --url "https://github.com/godatadriven/spot/releases/..." # TODO spot-complete-3.5_2.21.jar | |
| echo Run Ci from spark-opentelemetry directory | |
| ../../whirl ci | |
| whirl-ci-extra-env-spark-s3-to-postgres: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI non-default env | |
| working-directory: ./examples/spark-s3-to-postgres/ | |
| run: | | |
| echo Run Ci for example spark-s3-to-postgres with non-default env | |
| ../../whirl ci -e postgres-s3-spark | |
| whirl-ci-extra-env-api-to-s3: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| python_version: ["3.9", "3.12"] | |
| airflow_version: ["2.2.5", "2.9.1"] | |
| exclude: | |
| - airflow_version: "2.2.5" | |
| python_version: "3.12" | |
| runs-on: ubuntu-latest | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| AIRFLOW_VERSION: ${{ matrix.airflow_version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI api-to-s3 on k8s executor | |
| working-directory: ./examples/api-to-s3/ | |
| run: | | |
| echo Run Ci for example api-to-s3 with k8s executor env | |
| ../../whirl ci -e api-python-s3-k8s | |
| whirl-ci-extra-env-ha-scheduler: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| python_version: ["3.9", "3.12"] | |
| airflow_version: ["2.2.5", "2.9.1"] | |
| exclude: | |
| - airflow_version: "2.2.5" | |
| python_version: "3.12" | |
| runs-on: ubuntu-latest | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| AIRFLOW_VERSION: ${{ matrix.airflow_version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI ha-scheduler env | |
| working-directory: ./examples/external-airflow-db/ | |
| run: | | |
| echo Run Ci for example external-airflow-db with ha-scheduler env | |
| ../../whirl ci -e ha-scheduler |