Skip to content

Commit

Permalink
Make CI/CD parallel for the node-hub
Browse files Browse the repository at this point in the history
haixuanTao committed Nov 13, 2024
1 parent b460b9a commit ea2a527
Showing 2 changed files with 67 additions and 45 deletions.
66 changes: 45 additions & 21 deletions .github/workflows/node-hub-ci-cd.yml
Original file line number Diff line number Diff line change
@@ -11,9 +11,28 @@ on:
types: [published]

jobs:
ci:
find-jobs:
runs-on: ubuntu-latest
name: Find Jobs
outputs:
folders: ${{ steps.jobs.outputs.folders }}
steps:
- uses: actions/checkout@v1

- id: jobs
uses: kmanimaran/list-folder-action@v4
with:
path: ./node-hub

ci:
runs-on: ubuntu-latest
needs: [find-jobs]
defaults:
run:
working-directory: node-hub/${{ matrix.folder }}
strategy:
matrix:
folder: ${{ fromJson(needs.find-jobs.outputs.folders )}}
steps:
- name: Checkout repository
uses: actions/checkout@v2
@@ -53,11 +72,17 @@ jobs:

- name: Run Linting and Tests
run: |
chmod +x .github/workflows/node_hub_test.sh
.github/workflows/node_hub_test.sh
chmod +x ../../.github/workflows/node_hub_test.sh
../../.github/workflows/node_hub_test.sh
publish:
needs: [ci]
needs: [ci, find-jobs]
defaults:
run:
working-directory: ${{ matrix.folder }}
strategy:
matrix:
folder: ${{ fromJson(needs.find-jobs.outputs.folders )}}
runs-on: ubuntu-latest
if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')

@@ -102,21 +127,20 @@ jobs:
POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_PASS }}
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
run: |
for dir in node-hub/*/ ; do
if [ -d "$dir" ]; then
if [[ -f "$dir/Cargo.toml" && -f "$dir/pyproject.toml" ]]; then
echo "Publishing $dir using maturin..."
(cd "$dir" && poetry publish)
else
if [ -f "$dir/pyproject.toml" ]; then
echo "Publishing $dir using Poetry..."
(cd "$dir" && poetry publish --build)
fi
fi
if [ -f "$dir/Cargo.toml" ]; then
echo "Publishing $dir using Cargo..."
(cd "$dir" && cargo publish)
fi
dir=$(pwd)
base_dir=$(basename "$dir")
if [[ -f "Cargo.toml" && -f "pyproject.toml" ]]; then
echo "Publishing $dir using maturin..."
poetry publish
else
if [ -f "pyproject.toml" ]; then
echo "Publishing $dir using Poetry..."
poetry publish --build
fi
done
fi
if [ -f "Cargo.toml" ]; then
echo "Publishing $dir using Cargo..."
cargo publish
fi
46 changes: 22 additions & 24 deletions .github/workflows/node_hub_test.sh
Original file line number Diff line number Diff line change
@@ -4,29 +4,27 @@ set -euo
# List of ignored modules
ignored_folders=("dora-internvl" "dora-parler" "dora-keyboard" "dora-microphone" "terminal-input")

for dir in node-hub/*/ ; do
# Get the base name of the directory (without the path)
base_dir=$(basename "$dir")
# Get current working directory
dir=$(pwd)

# Check if the directory name is in the ignored list
if [[ " ${ignored_folders[@]} " =~ " ${base_dir} " ]]; then
echo "Skipping $base_dir as there is a hf model fetching issue..."
continue
fi
# Get the base name of the directory (without the path)
base_dir=$(basename "$dir")

if [ -d "$dir" ]; then
if [ -f "$dir/Cargo.toml" ]; then
echo "Running build and tests for Rust project in $dir..."
(cd "$dir" && cargo build)
(cd "$dir" && cargo test)
else
if [ -f "$dir/pyproject.toml" ]; then
echo "Running linting and tests for Python project in $dir..."
(cd "$dir" && pip install .)
(cd "$dir" && poetry run black --check .)
(cd "$dir" && poetry run pylint --disable=C,R --ignored-modules=cv2 **/*.py)
(cd "$dir" && poetry run pytest)
fi
fi
fi
done
# Check if the directory name is in the ignored list
if [[ " ${ignored_folders[@]} " =~ " ${base_dir} " ]]; then
echo "Skipping $base_dir as we cannot test it on the CI..."
else
if [ -f "$dir/Cargo.toml" ]; then
echo "Running build and tests for Rust project in $dir..."
cargo build
cargo test
else
if [ -f "$dir/pyproject.toml" ]; then
echo "Running linting and tests for Python project in $dir..."
pip install .
poetry run black --check .
poetry run pylint --disable=C,R --ignored-modules=cv2 **/*.py
poetry run pytest
fi
fi
fi

0 comments on commit ea2a527

Please sign in to comment.