update demo true #73
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Gather Sanity Checks | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: "0 20 * * *" | |
| jobs: | |
| gather: | |
| name: Gather Sanity Checks | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| - name: Get demo blueprint | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| mkdir -p sanitychecks/flows/blueprints | |
| rm -rf sanitychecks/flows/blueprints/*.yaml | |
| for flow in $(find flows -name "*.yaml"); do | |
| # Check if the file has extend.demo=true | |
| is_demo=$(yq eval '.extend.demo // false' "$flow") | |
| if [ "$is_demo" = "true" ]; then | |
| relative_path=${flow#flows/} | |
| target_dir="sanitychecks/flows/blueprints/$(dirname "$relative_path")" | |
| target_file="$target_dir/$(basename "$flow")" | |
| # mkdir -p "$target_dir" | |
| # Create a new YAML without the extend property and copy to "sanitychecks/flows/blueprints/ | |
| yq eval 'del(.extend) | .namespace = "sanitychecks"' "$flow" > "$target_file" | |
| echo "Copied $flow to $target_dir (without extend property)" | |
| fi | |
| done | |
| - name: Get all sanity check from remote repository | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| DESTINATION_PATH=src/test/resources/sanity-checks/ | |
| REPOSITORIES=$(gh repo list kestra-io -L 1000 --json name | jq -r '.[].name' | sort | grep "^plugin-"; echo "kestra") | |
| mkdir -p /tmp/repositories /tmp/results | |
| cd /tmp/repositories | |
| for REPOSITORY in $REPOSITORIES | |
| do | |
| git clone --filter=blob:none --sparse https://$GH_TOKEN:@github.com/kestra-io/$REPOSITORY.git | |
| cd $REPOSITORY | |
| git sparse-checkout init --cone | |
| # First, get a list of all directories in the repository | |
| git ls-tree -r --name-only HEAD | grep "/" > /tmp/all_paths | |
| # Find potential paths containing DESTINATION_PATH (without trailing slash) | |
| DESTINATION_NO_SLASH=${DESTINATION_PATH%/} | |
| POSSIBLE_PATHS=$(grep "/${DESTINATION_NO_SLASH}/" /tmp/all_paths || true) | |
| TOTAL_COUNT=0 | |
| FOUND_PATHS="" | |
| # If no paths found in git ls-tree, try the root path | |
| if [ -z "$POSSIBLE_PATHS" ]; then | |
| git sparse-checkout add "$DESTINATION_PATH" | |
| if [ -d "$DESTINATION_PATH" ]; then | |
| COUNT=$(find "$DESTINATION_PATH" -type f | wc -l) | |
| TOTAL_COUNT=$((TOTAL_COUNT + COUNT)) | |
| FOUND_PATHS="$DESTINATION_PATH" | |
| # Create results directory and copy files | |
| mkdir -p "../../results/$REPOSITORY" | |
| cp -R "$DESTINATION_PATH"/* "../../results/$REPOSITORY/" | |
| fi | |
| else | |
| # Create a temporary file to store unique directories | |
| echo "$POSSIBLE_PATHS" | while IFS= read -r PATH_ENTRY; do | |
| # Extract the base path up to sanity-checks | |
| BASE_PATH=$(echo "$PATH_ENTRY" | grep -o ".*${DESTINATION_NO_SLASH}") | |
| if [ ! -z "$BASE_PATH" ]; then | |
| echo "$BASE_PATH" | |
| fi | |
| done | sort -u > /tmp/unique_paths | |
| # Process each unique path | |
| while IFS= read -r UNIQUE_PATH; do | |
| git sparse-checkout add "$UNIQUE_PATH" | |
| if [ -d "$UNIQUE_PATH" ]; then | |
| COUNT=$(find "$UNIQUE_PATH" -type f | wc -l) | |
| if [ "$COUNT" -gt "0" ]; then | |
| TOTAL_COUNT=$((TOTAL_COUNT + COUNT)) | |
| # Append to FOUND_PATHS with a separator | |
| if [ -z "$FOUND_PATHS" ]; then | |
| FOUND_PATHS="$UNIQUE_PATH" | |
| else | |
| FOUND_PATHS="$FOUND_PATHS, $UNIQUE_PATH" | |
| fi | |
| # Create results directory and copy files | |
| mkdir -p "../../results/$REPOSITORY" | |
| cp -R "$UNIQUE_PATH"/* "../../results/$REPOSITORY/" | |
| fi | |
| fi | |
| done < /tmp/unique_paths | |
| fi | |
| if [ "$TOTAL_COUNT" -gt "0" ]; then | |
| echo -e "\x1B[32m-> $REPOSITORY - $TOTAL_COUNT found in paths: $FOUND_PATHS \x1B[0m" | |
| else | |
| echo -e "\x1B[31m-> $REPOSITORY - No sanity check found \x1B[0m" | |
| # Clean up empty results directory if nothing was found | |
| rm -rf "../../results/$REPOSITORY" | |
| fi | |
| cd .. | |
| rm -rf "$REPOSITORY" | |
| done | |
| rm -rf plugin-* /tmp/all_paths /tmp/unique_paths | |
| - name: Prepare the commit | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| mkdir -p sanitychecks/plugins | |
| rm -rf sanitychecks/plugins | |
| cp -R /tmp/results/ sanitychecks/plugins | |
| git add * | |
| - name: Check for changes and commit | |
| run: | | |
| if git diff --cached --quiet; then | |
| echo -e "\x1B[32m-> No changes to commit. Exiting with success. \x1B[0m" | |
| exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git commit -m "chore: add sanity check from plugin repository" | |
| git push origin main | |
| echo -e "\x1B[32m-> Pushing to main. \x1B[0m" |