sanity-check.yml

name: Gather Sanity Checks

on:
  push:
    branches:
      - main
  workflow_dispatch: {}
  schedule:
    - cron: "0 20 * * *"

jobs:
  gather:
    name: Gather Sanity Checks
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: main

      - name: Get demo blueprint
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          mkdir -p sanitychecks/flows/blueprints
          rm -rf sanitychecks/flows/blueprints/*.yaml
          for flow in $(find flows -name "*.yaml"); do
            # Check if the file has extend.demo=true
            is_demo=$(yq eval '.extend.demo // false' "$flow")
            
            if [ "$is_demo" = "true" ]; then
              relative_path=${flow#flows/}
              target_dir="sanitychecks/flows/blueprints/$(dirname "$relative_path")"
              target_file="$target_dir/$(basename "$flow")"
              
              # mkdir -p "$target_dir"
              
              # Create a new YAML without the extend property and copy to "sanitychecks/flows/blueprints/
              yq eval 'del(.extend) | .namespace = "sanitychecks"' "$flow" > "$target_file"
              echo "Copied $flow to $target_dir (without extend property)"
            fi
          done
      - name: Get all sanity check from remote repository
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          DESTINATION_PATH=src/test/resources/sanity-checks/
          REPOSITORIES=$(gh repo list kestra-io -L 1000 --json name | jq -r '.[].name' | sort | grep "^plugin-"; echo "kestra")

          mkdir -p /tmp/repositories /tmp/results
          cd /tmp/repositories

          for REPOSITORY in $REPOSITORIES
          do
              git clone --filter=blob:none --sparse https://$GH_TOKEN:@github.com/kestra-io/$REPOSITORY.git
              cd $REPOSITORY
              git sparse-checkout init --cone
              
              # First, get a list of all directories in the repository
              git ls-tree -r --name-only HEAD | grep "/" > /tmp/all_paths

              # Find potential paths containing DESTINATION_PATH (without trailing slash)
              DESTINATION_NO_SLASH=${DESTINATION_PATH%/}
              POSSIBLE_PATHS=$(grep "/${DESTINATION_NO_SLASH}/" /tmp/all_paths || true)
              TOTAL_COUNT=0
              FOUND_PATHS=""
              
              # If no paths found in git ls-tree, try the root path
              if [ -z "$POSSIBLE_PATHS" ]; then
                  git sparse-checkout add "$DESTINATION_PATH"
                  if [ -d "$DESTINATION_PATH" ]; then
                      COUNT=$(find "$DESTINATION_PATH" -type f | wc -l)
                      TOTAL_COUNT=$((TOTAL_COUNT + COUNT))
                      FOUND_PATHS="$DESTINATION_PATH"
                      
                      # Create results directory and copy files
                      mkdir -p "../../results/$REPOSITORY"
                      cp -R "$DESTINATION_PATH"/* "../../results/$REPOSITORY/"
                  fi
              else
                  # Create a temporary file to store unique directories
                  echo "$POSSIBLE_PATHS" | while IFS= read -r PATH_ENTRY; do
                      # Extract the base path up to sanity-checks
                      BASE_PATH=$(echo "$PATH_ENTRY" | grep -o ".*${DESTINATION_NO_SLASH}")
                      if [ ! -z "$BASE_PATH" ]; then
                          echo "$BASE_PATH"
                      fi
                  done | sort -u > /tmp/unique_paths
                  
                  # Process each unique path
                  while IFS= read -r UNIQUE_PATH; do
                      git sparse-checkout add "$UNIQUE_PATH"
                      
                      if [ -d "$UNIQUE_PATH" ]; then
                          COUNT=$(find "$UNIQUE_PATH" -type f | wc -l)
                          if [ "$COUNT" -gt "0" ]; then
                              TOTAL_COUNT=$((TOTAL_COUNT + COUNT))
                              # Append to FOUND_PATHS with a separator
                              if [ -z "$FOUND_PATHS" ]; then
                                  FOUND_PATHS="$UNIQUE_PATH"
                              else
                                  FOUND_PATHS="$FOUND_PATHS, $UNIQUE_PATH"
                              fi
                              
                              # Create results directory and copy files
                              mkdir -p "../../results/$REPOSITORY"
                              cp -R "$UNIQUE_PATH"/* "../../results/$REPOSITORY/"
                          fi
                      fi
                  done < /tmp/unique_paths
              fi

              if [ "$TOTAL_COUNT" -gt "0" ]; then
                  echo -e "\x1B[32m-> $REPOSITORY - $TOTAL_COUNT found in paths: $FOUND_PATHS \x1B[0m"
              else
                  echo -e "\x1B[31m-> $REPOSITORY - No sanity check found \x1B[0m"
                  # Clean up empty results directory if nothing was found
                  rm -rf "../../results/$REPOSITORY"
              fi

              cd ..
              rm -rf "$REPOSITORY"
          done

          rm -rf plugin-* /tmp/all_paths /tmp/unique_paths

      - name: Prepare the commit
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          mkdir -p sanitychecks/plugins
          rm -rf sanitychecks/plugins
          cp -R /tmp/results/ sanitychecks/plugins
          git add *

      - name: Check for changes and commit
        run: |
          if git diff --cached --quiet; then
              echo -e "\x1B[32m-> No changes to commit. Exiting with success. \x1B[0m"
              exit 0
          fi

          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git commit -m "chore: add sanity check from plugin repository"
          git push origin main

          echo -e "\x1B[32m-> Pushing to main. \x1B[0m"