Publish rust project on pip to make it simpler to deploy dora node on…

… different machine without requiring installing cargo (#695) Currently rust node and the cli requires easier to use cargo or wget, but it would be simpler if we could just pip install them. This PR do this. People will be able to do: ```bash pip install dora-rerun pip install dora-rs-cli ``` This will make the pull process of nodes simpler for people who only use python.
dora-rs · Nov 1, 2024 · af620b4 · af620b4
2 parents 20c6328 + dc716b2
commit af620b4
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 37 deletions.
diff --git a/.github/workflows/node-hub-ci-cd.yml b/.github/workflows/node-hub-ci-cd.yml
@@ -104,9 +104,14 @@ jobs:
         run: |
           for dir in node-hub/*/ ; do
             if [ -d "$dir" ]; then
-              if [ -f "$dir/pyproject.toml" ]; then
-                echo "Publishing $dir using Poetry..."
-                (cd "$dir" && poetry publish --build)
+              if [[ -f "$dir/Cargo.toml" && -f "$dir/pyproject.toml" ]]; then
+                echo "Publishing $dir using maturin..."
+                (cd "$dir" && poetry publish)
+              else
+                if [ -f "$dir/pyproject.toml" ]; then
+                  echo "Publishing $dir using Poetry..."
+                  (cd "$dir" && poetry publish --build)
+                fi
               fi
               
               if [ -f "$dir/Cargo.toml" ]; then

diff --git a/.github/workflows/node_hub_test.sh b/.github/workflows/node_hub_test.sh
@@ -15,18 +15,18 @@ for dir in node-hub/*/ ; do
     fi
 
     if [ -d "$dir" ]; then
-        if [ -f "$dir/pyproject.toml" ]; then
-        echo "Running linting and tests for Python project in $dir..."
-        (cd "$dir" && pip install .)
-        (cd "$dir" && poetry run black --check .)
-        (cd "$dir" && poetry run pylint --disable=C,R  --ignored-modules=cv2 **/*.py)
-        (cd "$dir" && poetry run pytest)
-        fi
-
         if [ -f "$dir/Cargo.toml" ]; then
-        echo "Running build and tests for Rust project in $dir..."
-        (cd "$dir" && cargo build)
-        (cd "$dir" && cargo test)
-        fi
+            echo "Running build and tests for Rust project in $dir..."
+            (cd "$dir" && cargo build)
+            (cd "$dir" && cargo test)
+        else
+            if [ -f "$dir/pyproject.toml" ]; then
+            echo "Running linting and tests for Python project in $dir..."
+            (cd "$dir" && pip install .)
+            (cd "$dir" && poetry run black --check .)
+            (cd "$dir" && poetry run pylint --disable=C,R  --ignored-modules=cv2 **/*.py)
+            (cd "$dir" && poetry run pytest)
+            fi
+        fi   
     fi
 done
diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml
@@ -39,6 +39,9 @@ jobs:
           # target: s390x
           # - runner: ubuntu-20.04
           # target: ppc64le
+        repository:
+          - path: apis/python/node
+          - path: binaries/cli
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -53,21 +56,21 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist --zig
+          args: --release --out dist --zig -i 3.8
           manylinux: manylinux_2_28
-          working-directory: apis/python/node
+          working-directory: ${{ matrix.repository.path }}
       - name: Upload wheels
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v4
         with:
           name: wheels-linux-${{ matrix.platform.target }}
-          path: apis/python/node/dist
+          path: ${{ matrix.repository.path }}/dist
       - name: Upload to release
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: apis/python/node/dist/*
+          file: ${{ matrix.repository.path }}/dist/*
           tag: ${{ github.ref }}
           file_glob: true
 
@@ -82,6 +85,9 @@ jobs:
             target: x86
           - runner: ubuntu-22.04
             target: aarch64
+        repository:
+          - path: apis/python/node
+          - path: binaries/cli
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -91,22 +97,22 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist
+          args: --release --out dist -i 3.8
           sccache: "true"
           manylinux: musllinux_1_2
-          working-directory: apis/python/node
+          working-directory: ${{ matrix.repository.path }}
       - name: Upload wheels
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v4
         with:
           name: wheels-musllinux-${{ matrix.platform.target }}
-          path: apis/python/node/dist
+          path: ${{ matrix.repository.path }}/dist
       - name: Upload to release
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: apis/python/node/dist/*
+          file: ${{ matrix.repository.path }}/dist/*
           tag: ${{ github.ref }}
           file_glob: true
 
@@ -121,6 +127,9 @@ jobs:
               image_tag: "armv7-musleabihf",
             },
           ]
+        repository:
+          - path: apis/python/node
+          - path: binaries/cli
     container:
       image: docker://messense/rust-musl-cross:${{ matrix.platform.image_tag }}
       env:
@@ -136,20 +145,20 @@ jobs:
           target: ${{ matrix.platform.target }}
           manylinux: auto
           container: off
-          args: --release -o dist
-          working-directory: apis/python/node
+          args: --release -o dist -i 3.8
+          working-directory: ${{ matrix.repository.path }}
       - name: Upload wheels
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v3
         with:
           name: wheels-musllinux-${{ matrix.platform.target }}
-          path: apis/python/node/dist
+          path: ${{ matrix.repository.path }}/dist
       - name: Upload to release
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: apis/python/node/dist/*
+          file: ${{ matrix.repository.path }}/dist/*
           tag: ${{ github.ref }}
           file_glob: true
 
@@ -160,6 +169,9 @@ jobs:
         platform:
           - runner: windows-latest
             target: x64
+        repository:
+          - path: apis/python/node
+          - path: binaries/cli
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -170,21 +182,21 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist
+          args: --release --out dist -i 3.8
           sccache: "true"
-          working-directory: apis/python/node
+          working-directory: ${{ matrix.repository.path }}
       - name: Upload wheels
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v4
         with:
           name: wheels-windows-${{ matrix.platform.target }}
-          path: apis/python/node/dist
+          path: ${{ matrix.repository.path }}/dist
       - name: Upload to release
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: apis/python/node/dist/*
+          file: ${{ matrix.repository.path }}/dist/*
           tag: ${{ github.ref }}
           file_glob: true
 
@@ -197,6 +209,9 @@ jobs:
             target: x86_64
           - runner: macos-14
             target: aarch64
+        repository:
+          - path: apis/python/node
+          - path: binaries/cli
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -206,40 +221,45 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist
+          args: --release --out dist -i 3.8
           sccache: "true"
-          working-directory: apis/python/node
+          working-directory: ${{ matrix.repository.path }}
       - name: Upload wheels
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v4
         with:
           name: wheels-macos-${{ matrix.platform.target }}
-          path: apis/python/node/dist
+          path: ${{ matrix.repository.path }}/dist
       - name: Upload to release
         if: github.event_name == 'release'
         uses: svenstaro/upload-release-action@v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: apis/python/node/dist/*
+          file: ${{ matrix.repository.path }}/dist/*
           tag: ${{ github.ref }}
           file_glob: true
 
   sdist:
     runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        repository:
+          - path: apis/python/node
+          - path: binaries/cli
     steps:
       - uses: actions/checkout@v3
       - name: Build sdist
         uses: PyO3/maturin-action@v1
         with:
           command: sdist
           args: --out dist
-          working-directory: apis/python/node
+          working-directory: ${{ matrix.repository.path }}
       - name: Upload sdist
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v3
         with:
           name: wheels
-          path: apis/python/node/dist
+          path: ${{ matrix.repository.path }}/dist
 
   release:
     name: Release

diff --git a/binaries/cli/pyproject.toml b/binaries/cli/pyproject.toml
@@ -0,0 +1,6 @@
+[build-system]
+requires = ["maturin>=0.13.2"]
+build-backend = "maturin"
+
+[project]
+name = "dora-rs-cli"
diff --git a/node-hub/dora-rerun/pyproject.toml b/node-hub/dora-rerun/pyproject.toml
@@ -0,0 +1,6 @@
+[build-system]
+requires = ["maturin>=0.13.2"]
+build-backend = "maturin"
+
+[project]
+name = "dora-rerun"