[ci] Automate release publishing

.github/workflows/release.yml

@@ -1,40 +1,36 @@
 name: Publishing Release
 on:
-  release:
-    # https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows#release
-    types: [published]
-  # When triggered by schedule and workflow_dispatch, github.event.action is an empty string.
-  # We use this to distinguish which taichi to release.
   schedule:
     - cron: "0 0 * * *"
   workflow_dispatch:
+    # Manually trigger the release workflow, a version must be provided
+    inputs:
+      version:
+        description: "The version to release (e.g. v0.8.0)"
+        type: string
+        required: true
 
 env:
   PROD_PWD: ${{ secrets.PYPI_PWD_PROD }}
   NIGHT_PWD: ${{ secrets.PYPI_PWD_NIGHTLY }}
   METADATA_USERNAME: ${{ secrets.METADATA_USERNAME }}
   METADATA_PASSWORD: ${{ secrets.METADATA_PASSWORD }}
   METADATA_URL: ${{ secrets.METADATA_URL }}
+  RELEASE_VERSION: ${{ github.event.inputs.version }}
 
 jobs:
   add_version_to_database:
     name: Add version to database
     # Skip running release workflow on forks
-    if: github.repository_owner == 'taichi-dev'
+    if: github.repository_owner == 'taichi-dev' && github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
 
-      - name: set tag
-        id: tag
-        run: echo ::set-output name=version::${GITHUB_REF#refs/*/}
-
       - name: Save new version
         run: |
           python3 -m pip install requests==2.26
-          [ -z "${{ github.event.action }}" ] || python3 misc/save_new_version.py
-        env:
-          RELEASE_VERSION: ${{ steps.tag.outputs.version }}
+          python3 misc/save_new_version.py
 
   # This job set environment matrix with respect to production release and nightly release.
   matrix_prep:
@@ -46,20 +42,20 @@ jobs:
     steps:
       - id: set-matrix
         run: |
-          if [ -z "${{ github.event.action }}" ]; then
+          if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then
+            # For production release, we run on four python versions.
+            echo '::set-output name=matrix::{"include":[{"name":"taichi","python":"3.6","conda_python":"py36"},{"name":"taichi","python":"3.7","conda_python":"py37"},{"name":"taichi","python":"3.8","conda_python":"py38"},{"name":"taichi","python":"3.9","conda_python":"py39"}]}"'
+
+            echo '::set-output name=matrix_osx::{"include":[{"name":"taichi","python":"3.8"},{"name":"taichi","python":"3.9"}]}"'
+          else
             # For nightly release, we only run on python 3.8
             echo '::set-output name=matrix::{"include":[{"name":"taichi-nightly","python":"3.8","conda_python":"py38"},{"name":"taichi-nightly","python":"3.10","conda_python":"py310"}]}"'
 
             # M1 only supports py38 and py39(conda), so change matrix.
             echo '::set-output name=matrix_osx::{"include":[{"name":"taichi-nightly","python":"3.8"},{"name":"taichi-nightly","python":"3.10"}]}"'
-          else
-            # For production release, we run on four python versions.
-            echo '::set-output name=matrix::{"include":[{"name":"taichi","python":"3.6","conda_python":"py36"},{"name":"taichi","python":"3.7","conda_python":"py37"},{"name":"taichi","python":"3.8","conda_python":"py38"},{"name":"taichi","python":"3.9","conda_python":"py39"}]}"'
-
-            echo '::set-output name=matrix_osx::{"include":[{"name":"taichi","python":"3.8"},{"name":"taichi","python":"3.9"}]}"'
           fi
 
-  build_and_upload_linux:
+  build_and_test_linux:
     name: Build and Upload (linux only)
     needs: matrix_prep
     strategy:
@@ -85,11 +81,11 @@ jobs:
           docker create --user dev --name taichi_build --gpus all -v /tmp/.X11-unix:/tmp/.X11-unix \
             -e DISPLAY -e PY -e GPU_BUILD -e TAICHI_CMAKE_ARGS -e PROJECT_NAME \
             registry.taichigraphics.com/taichidev-ubuntu18.04:v0.2.1 \
-            /home/dev/taichi/.github/workflows/scripts/unix_build.sh
+            /home/dev/${{ github.event.repository.name }}/.github/workflows/scripts/unix_build.sh
           tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - taichi_build:/home/dev/
           docker start -a taichi_build
-          docker cp taichi_build:/home/dev/taichi/dist shared/dist
-          docker cp taichi_build:/home/dev/taichi/build shared/build
+          docker cp taichi_build:/home/dev/${{ github.event.repository.name }}/dist shared/dist
+          docker cp taichi_build:/home/dev/${{ github.event.repository.name }}/build shared/build
         env:
           PY: ${{ matrix.conda_python }}
           GPU_BUILD: ON
@@ -98,9 +94,9 @@ jobs:
           DISPLAY: ":1"
 
       - name: Archive Wheel Artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.name }}-py${{ matrix.python }}-linux.whl
+          name: dist
           path: shared/dist/*.whl
           retention-days: 20
 
@@ -120,20 +116,12 @@ jobs:
           GPU_TEST: ON
           DISPLAY: ":1"
 
-      - name: Upload PyPI
-        env:
-          PROJECT_NAME: ${{ matrix.name }}
-        run: |
-          cd shared
-          pip install twine requests==2.26
-          python3 ../misc/upload_release.py
-
       - name: clean docker container
         if: always()
         run: |
           docker rm taichi_build taichi_test -f
 
-  build_and_upload_mac:
+  build_and_test_mac:
     name: Build and Upload (macOS only)
     needs: matrix_prep
     strategy:
@@ -174,9 +162,9 @@ jobs:
           CXX: clang++
 
       - name: Archive Wheel Artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.name }}-py${{ matrix.python }}-macos.whl
+          name: dist
           path: dist/*.whl
           retention-days: 20
 
@@ -185,13 +173,7 @@ jobs:
         env:
           TI_WANTED_ARCHS: "cpu"
 
-      - name: Upload PyPI
-        env:
-          # https://docs.github.com/en/free-pro-team@latest/actions/reference/encrypted-secrets#using-encrypted-secrets-in-a-workflow
-          PROJECT_NAME: ${{ matrix.name }}
-        run: python misc/upload_release.py
-
-  build_and_upload_m1:
+  build_and_test_m1:
     name: Build and Upload (Apple M1)
     needs: matrix_prep
     strategy:
@@ -230,9 +212,9 @@ jobs:
           CXX: clang++
 
       - name: Archive Wheel Artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.name }}-py${{ matrix.python }}-macos-m1.whl
+          name: dist
           path: dist/*.whl
           retention-days: 20
 
@@ -245,15 +227,7 @@ jobs:
           PYTHON: ${{ matrix.python }}
           GPU_TEST: ON
 
-      - name: Upload PyPI
-        env:
-          PROJECT_NAME: ${{ matrix.name }}
-          PYTHON: ${{ matrix.python }}
-        run: |
-          export PATH=/Users/github/miniforge3/envs/$PYTHON/bin:$PATH
-          python misc/upload_release.py
-
-  build_and_upload_macos_1014:
+  build_and_test_macos_1014:
     name: Build and Upload (macos 1014)
     needs: matrix_prep
     strategy:
@@ -289,9 +263,9 @@ jobs:
           CXX: clang++
 
       - name: Archive Wheel Artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.name }}-py${{ matrix.python }}-macos-1014.whl
+          name: dist
           path: dist/*.whl
           retention-days: 20
 
@@ -303,15 +277,7 @@ jobs:
           TI_WANTED_ARCHS: "cpu"
           PYTHON: ${{ matrix.python }}
 
-      - name: Upload PyPI
-        env:
-          PROJECT_NAME: ${{ matrix.name }}
-          PYTHON: ${{ matrix.python }}
-        run: |
-          export PATH=/Users/buildbot6/miniconda3/envs/$PYTHON/bin:$PATH
-          python misc/upload_release.py
-
-  build_and_upload_windows:
+  build_and_test_windows:
     name: Build and Upload (Windows only)
     needs: matrix_prep
     strategy:
@@ -351,7 +317,7 @@ jobs:
       - name: Archive Wheel Artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ matrix.name }}-py${{ matrix.python }}-windows.whl
+          name: dist
           path: dist/*.whl
           retention-days: 20
 
@@ -367,10 +333,90 @@ jobs:
         env:
           TI_SKIP_VERSION_CHECK: ON
 
-      - name: Upload PyPI
-        shell: powershell
-        env:
-          PROJECT_NAME: ${{ matrix.name }}
+  upload_to_pypi:
+    name: Upload release to PyPI
+    needs:
+      [
+        build_and_test_linux,
+        build_and_test_mac,
+        build_and_test_m1,
+        build_and_test_macos_1014,
+        build_and_test_windows,
+      ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Get dist files
+        uses: actions/download-artifact@v3
+        with:
+          name: dist
+          path: dist
+
+      - name: Upload to PyPI
+        run: |
+          ls -l dist/
+          if [ -z "$RELEASE_VERSION" ]; then
+            export PROJECT_NAME="taichi-nightly"
+          else
+            export PROJECT_NAME="taichi"
+          fi
+          python -m pip install requests twine
+          python misc/upload_release.py
+
+  create_release:
+    name: Create tag and publish release
+    needs: upload_to_pypi
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Generate Changelog
+        id: changelog
+        run: |
+          pip3 install gitpython
+          content=$(python3 misc/make_changelog.py)
+          echo $content
+          # Escape multiline strings:
+          # https://renehernandez.io/snippets/multiline-strings-as-a-job-output-in-github-actions/
+          content="${content//'%'/'%25'}"
+          content="${content//$'\n'/'%0A'}"
+          content="${content//$'\r'/'%0D'}"
+          echo "::set-output name=content::$content"
+
+      - name: Create tag
+        run: |
+          git config user.email "[email protected]"
+          git config user.name "Taichi Gardener"
+          git tag -a ${RELEASE_VERSION} -m "Release ${RELEASE_VERSION}"
+          git push origin --tags
+
+      - name: Publish release
+        uses: softprops/action-gh-release@v1
+        with:
+          body: ${{ steps.changelog.outputs.content }}
+          tag_name: ${{ github.event.inputs.version }}
+
+      - name: Bump version
         run: |
-          python -m pip install twine
-          venv\Scripts\python misc/upload_release.py
+          version_parts=(${RELEASE_VERSION//./ })
+          version_parts[2]=$(expr ${version_parts[2]} + 1)
+          next_version=$(IFS=.; echo "${version_parts[*]}")
+          # Update version.txt
+          git checkout -b "bump/$next_version"
+          echo "$next_version" > version.txt
+          git add version.txt
+          # Commit and push changes
+          git commit -m "Bump version to $next_version"
+          git push origin "bump/$next_version"
+          # Create pull request
+          gh pr create -B master -t "[misc] Bump version to $next_version"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GARDENER_PAT }}

MANIFEST.in

@@ -1,4 +1,5 @@
 include MANIFEST.in
+include version.txt
 include python/*.txt
 include python/*.py
 include *.cfg

misc/upload_release.py

@@ -9,36 +9,41 @@ def upload_taichi_version():
     username = os.getenv('METADATA_USERNAME')
     password = os.getenv('METADATA_PASSWORD')
     url = os.getenv('METADATA_URL')
-    filename = os.listdir('./dist')[0]
-    filename = filename[:len(filename) - 4]
-    parts = filename.split('-')
-    payload = {'version': parts[1], 'platform': parts[4], 'python': parts[2]}
-    try:
-        response = requests.post(f'https://{url}/add_version/detail',
-                                 json=payload,
-                                 auth=(username, password),
-                                 timeout=5)
-        response.raise_for_status()
-    except requests.exceptions.ConnectionError as err:
-        sys.exit('Updating latest version failed: No internet, ' + str(err))
-    except requests.exceptions.HTTPError as err:
-        sys.exit('Updating latest version failed: Server error, ' + str(err))
-    except requests.exceptions.Timeout as err:
-        sys.exit(
-            'Updating latest version failed: Time out when connecting server, '
-            + str(err))
-    except requests.exceptions.RequestException as err:
-        sys.exit('Updating latest version failed: ' + str(err))
-    response = response.json()
-    print(response['message'])
+    for filename in os.listdir('./dist'):
+        filename = filename[:len(filename) - 4]
+        parts = filename.split('-')
+        payload = {
+            'version': parts[1],
+            'platform': parts[4],
+            'python': parts[2]
+        }
+        try:
+            response = requests.post(f'https://{url}/add_version/detail',
+                                     json=payload,
+                                     auth=(username, password),
+                                     timeout=5)
+            response.raise_for_status()
+        except requests.exceptions.ConnectionError as err:
+            print('Updating latest version failed: No internet,', err)
+        except requests.exceptions.HTTPError as err:
+            print('Updating latest version failed: Server error,', err)
+        except requests.exceptions.Timeout as err:
+            print(
+                'Updating latest version failed: Time out when connecting server,',
+                err)
+        except requests.exceptions.RequestException as err:
+            print('Updating latest version failed:', err)
+        else:
+            response = response.json()
+            print(response['message'])
 
 
 def upload_artifact(is_taichi):
     pwd_env = 'PROD_PWD' if is_taichi else 'NIGHT_PWD'
     twine_password = os.getenv(pwd_env)
     if not twine_password:
         sys.exit(f'Missing password env var {pwd_env}')
-    command = ["python3", "-m", "twine", "upload"]
+    command = [sys.executable, '-m', 'twine', 'upload']
     if not is_taichi:
         command.extend(['--repository', 'testpypi'])
     command.extend(
@@ -50,6 +55,10 @@ def upload_artifact(is_taichi):
 
 
 if __name__ == '__main__':
+    if os.getenv('GITHUB_REPOSITORY',
+                 'taichi-dev/taichi') != 'taichi-dev/taichi':
+        print('This script should be run from taichi repo')
+        sys.exit(0)
     is_taichi = os.getenv('PROJECT_NAME', 'taichi') == 'taichi'
     upload_artifact(is_taichi)
     if is_taichi: