checks.yml

name: Master

on:
  push:
    branches:
      - master
    paths-ignore:
      - "docs/**"
  pull_request:
    paths-ignore:
      - "docs/**"
jobs:
  lint:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11"]
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements files
          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt')) }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r dev-requirements.txt
      - name: Pip info
        run: python -m pip list
      - name: Lint
        run: pre-commit run --all --show-diff-on-failure

  # This is the build system for the new example directory structure
  list_examples:
    runs-on: ubuntu-latest
    name: "Create a list of example packages"
    steps:
      - uses: actions/checkout@v2
      - name: "Provide the list"
        id: create-example-list
        run: echo "PACKAGES=$(find examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT"
    outputs:
      packages: "${{ steps.create-example-list.outputs.PACKAGES }}"

  trigger_serialize_register_examples:
    name: Serialize & Register Flytesnacks workflow
    needs: [list_examples]
    uses: ./.github/workflows/serialize_example.yml
    with:
      packages: ${{ needs.list_examples.outputs.PACKAGES }}
    secrets:
      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}

  push_example_image_to_github:
    name: Build & Push Example Image to GHCR
    runs-on: ubuntu-latest
    needs: [list_examples]
    strategy:
      matrix:
        directory: "${{ fromJson(needs.list_examples.outputs.packages) }}"
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: "0"
      - name: "Set example name"
        id: example_id
        run: echo "EXAMPLE_NAME=$(basename -- ${{ matrix.directory }})" >> "$GITHUB_OUTPUT"
      - name: Build & Push Docker Image to Github Registry
        uses: whoan/docker-build-with-cache-action@v5
        with:
          # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package
          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
          password: "${{ secrets.FLYTE_BOT_PAT }}"
          image_name: ${{ github.repository_owner }}/flytecookbook
          image_tag: ${{ steps.example_id.outputs.EXAMPLE_NAME }}-latest,${{ steps.example_id.outputs.EXAMPLE_NAME }}-${{ github.sha }}
          registry: ghcr.io
          push_git_tag: ${{ github.event_name != 'pull_request' }}
          push_image_and_stages: ${{ github.event_name != 'pull_request' }}
          build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:${{ steps.example_id.outputs.id }}-${{ github.sha }}"
          context: ${{ matrix.directory }}
          dockerfile: Dockerfile

  bump_version:
    name: Bump Version
    if: ${{ github.event_name != 'pull_request' }}
    needs: [trigger_serialize_register_examples, push_example_image_to_github] # Only to ensure it can successfully build
    uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master
    secrets:
      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}

  prerelease:
    name: Create Prerelease
    needs: [bump_version]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: "0"
      - name: Create Pre release
        id: prerelease
        run: |
          RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \
          --header 'Accept: application/vnd.github.v3+json' \
          --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
          --data-raw '{
              "tag_name": "'${{ needs.bump-version.outputs.version }}'",
              "prerelease": true
          }' | jq -r '.id')
          echo ::set-output name=release_id::$RELEASE_ID
    outputs:
      release_id: ${{ steps.prerelease.outputs.release_id }}

  # Download artifacts again and push them to the release only if this is not a pull request
  release_workflow:
    name: Publish artifacts to github release
    runs-on: ubuntu-latest
    needs: [prerelease]
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: "0"
      - name: setup download artifact dir
        run: |
          mkdir download-artifact
      - name: Download artifacts
        uses: actions/download-artifact@v2
        with:
          path: ./download-artifact/
      - name: Package Examples
        run: |
          mkdir -p release-snacks
          cd download-artifact
          for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait
          cd .. && sudo rm -rf download-artifact/
          cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json
      - name: Release test manifest
        uses: goreleaser/goreleaser-action@v2
        with:
          version: latest
          args: release --rm-dist
        env:
          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
          GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }}

  make_release:
    name: Mark github pre-release as Release
    needs: [release_workflow]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: "0"
      - name: Update Release
        id: release
        run: |
          curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \
          --header 'Accept: application/vnd.github.v3+json' \
          --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
          --data-raw '{
              "tag_name": "'${{ needs.bump-version.outputs.version }}'",
              "prerelease": false
          }'

  e2e-tests:
    runs-on: ubuntu-latest
    env:
      FLYTESNACKS_PRIORITIES: "P0"
      FLYTESNACKS_VERSION: ""
    timeout-minutes: 30
    steps:
      - name: Set latest Flytesnacks release
        if: ${{ env.FLYTESNACKS_VERSION == '' }}
        run: |
          FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)"
          echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      - uses: unionai/flytectl-setup-action@v0.0.1
      - name: Setup sandbox
        run: |
          mkdir -p ~/.flyte/sandbox
          cat << EOF > ~/.flyte/sandbox/config.yaml
          task_resources:
            defaults:
              cpu: "0"
              memory: "0"
            limits:
              cpu: "0"
              memory: "0"
          EOF
          flytectl demo start --imagePullPolicy Never
      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          pip install flytekit flytekitplugins-deck-standard
          pip freeze
      - name: Checkout flytesnacks
        uses: actions/checkout@v3
        with:
          repository: flyteorg/flytesnacks
          path: flytesnacks
      - name: Register specific tests
        run: |
          for f in \
                  basics/basics/hello_world.py \
                  basics/basics/workflow.py \
                  basics/basics/named_outputs.py \
                  advanced_composition/advanced_composition/chain_entities.py \
                  advanced_composition/advanced_composition/dynamics.py \
                  advanced_composition/advanced_composition/map_task.py \
                  advanced_composition/advanced_composition/subworkflows.py \
                  data_types_and_io/data_types_and_io/dataclass.py \
                  data_types_and_io/data_types_and_io/structured_dataset.py ;
          do
              pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                  register \
                  --project flytesnacks \
                  --domain development \
                  --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
                  --version ${{ env.FLYTESNACKS_VERSION }} \
                  flytesnacks/examples/$f;
          done
      - name: End2End
        run: |
          make end2end_execute