From ba2b4ea70c2235b46058a843c98ef73239790b58 Mon Sep 17 00:00:00 2001
From: Proton <feisuzhu@163.com>
Date: Mon, 22 Aug 2022 16:38:02 +0800
Subject: [PATCH] [ci] Add more release tests

---
 .github/workflows/scripts/unix_test.sh | 23 +++++++++++++++++++----
 .github/workflows/scripts/win_test.ps1 | 16 ++++++++++++----
 .github/workflows/testing.yml          | 20 ++++++++++++++++++--
 3 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/scripts/unix_test.sh b/.github/workflows/scripts/unix_test.sh
index ea8a0ae4e7a6d..cb34dcb1058ba 100755
--- a/.github/workflows/scripts/unix_test.sh
+++ b/.github/workflows/scripts/unix_test.sh
@@ -17,7 +17,7 @@ setup_python
 python3 -m pip install dist/*.whl
 if [ -z "$GPU_TEST" ]; then
     python3 -m pip install -r requirements_test.txt
-    python3 -m pip install "torch; python_version < '3.10'"
+    python3 -m pip install "torch>1.12.0; python_version < '3.10'"
     # Paddle's develop package doesn't support CI's MACOS machine at present
     if [[ $OSTYPE == "linux-"* ]]; then
         python3 -m pip install "paddlepaddle==2.3.0; python_version < '3.10'"
@@ -47,6 +47,8 @@ echo "wanted archs: $TI_WANTED_ARCHS"
 if [ "$TI_RUN_RELEASE_TESTS" == "1" -a -z "$TI_LITE_TEST" ]; then
     python3 -m pip install PyYAML
     git clone https://github.com/taichi-dev/taichi-release-tests
+    pushd taichi-release-tests
+    git checkout v1.1.0
     mkdir -p repos/taichi/python/taichi
     EXAMPLES=$(cat <<EOF | python3 | tail -n 1
 import taichi.examples
@@ -54,10 +56,23 @@ print(taichi.examples.__path__[0])
 EOF
 )
     ln -sf $EXAMPLES repos/taichi/python/taichi/examples
-    ln -sf taichi-release-tests/truths truths
-    python3 taichi-release-tests/run.py --log=DEBUG --runners 1 taichi-release-tests/timelines
-fi
+    pushd repos
+    git clone --depth=1 https://github.com/taichi-dev/quantaichi
+    git clone --depth=1 https://github.com/taichi-dev/difftaichi
+    popd
+
+    pushd repos/difftaichi
+    if [ "$(uname -s):$(uname -m)" == "Darwin:arm64" ]; then
+        # No FORTRAN compiler is currently working reliably on M1 Macs
+        # We can't just pip install scipy, using conda instead
+        conda install -y scipy
+    fi
+    pip install -r requirements.txt
+    popd
 
+    python3 run.py --log=DEBUG --runners 1 timelines
+    popd
+fi
 
 if [ -z "$TI_SKIP_CPP_TESTS" ]; then
     python3 tests/run_tests.py --cpp
diff --git a/.github/workflows/scripts/win_test.ps1 b/.github/workflows/scripts/win_test.ps1
index 36659f62187e8..27b23a4f3eda0 100644
--- a/.github/workflows/scripts/win_test.ps1
+++ b/.github/workflows/scripts/win_test.ps1
@@ -61,13 +61,21 @@ if ("$env:TI_WANTED_ARCHS".Contains("vulkan")) {
 Invoke python tests/run_tests.py -vr2 -t1 -k "torch" -a "$env:TI_WANTED_ARCHS"
 
 if ("$env:TI_RUN_RELEASE_TESTS" -eq "1" -and -not "$env:TI_LITE_TEST") {
-    echo "Running release tests"
+    Info "Running release tests"
     # release tests
     Invoke pip install PyYAML
     Invoke git clone https://github.com/taichi-dev/taichi-release-tests
+    Push-Location taichi-release-tests
+    Invoke git checkout v1.1.0
     mkdir -p repos/taichi/python/taichi
     $EXAMPLES = & python -c 'import taichi.examples as e; print(e.__path__._path[0])' | Select-Object -Last 1
-    New-Item -Target $EXAMPLES -Path repos/taichi/python/taichi/examples -ItemType Junction
-    New-Item -Target taichi-release-tests/truths -Path truths -ItemType Junction
-    Invoke python taichi-release-tests/run.py --log=DEBUG --runners 1 taichi-release-tests/timelines
+    Push-Location repos
+    Invoke git clone --depth=1 https://github.com/taichi-dev/quantaichi
+    Invoke git clone --depth=1 https://github.com/taichi-dev/difftaichi
+    Pop-Location
+    Push-Location repos/difftaichi
+    Invoke pip install -r requirements.txt
+    Pop-Location
+    Invoke python run.py --log=DEBUG --runners 1 timelines
+    Pop-Location
 }
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index dec18a9236e8b..79b3ee76ff91d 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -394,7 +394,7 @@ jobs:
         env:
           PY: py38
           TI_WANTED_ARCHS: 'cuda,vulkan,opengl'
-          TI_DEVICE_MEMORY_GB: '0.7'
+          TI_DEVICE_MEMORY_GB: '1'
           TI_RUN_RELEASE_TESTS: '1'
 
       - name: Save wheel if test failed
@@ -405,6 +405,14 @@ jobs:
           path: dist/*
           retention-days: 7
 
+      - name: Save Bad Captures
+        if: failure() && steps.test.conclusion == 'failure'
+        uses: actions/upload-artifact@v3
+        with:
+          name: bad-captures
+          path: taichi-release-tests/bad-compare/*
+          retention-days: 7
+
   build_and_test_windows:
     name: Build and Test Windows
     needs: check_files
@@ -456,7 +464,7 @@ jobs:
           PY: "3.7"
           TI_WANTED_ARCHS: ${{ matrix.archs }}
           TI_SKIP_VERSION_CHECK: ON
-          TI_DEVICE_MEMORY_GB: '0.7'
+          TI_DEVICE_MEMORY_GB: '1'
           TI_RUN_RELEASE_TESTS: '1'
 
       - name: Save wheel if test failed
@@ -524,6 +532,14 @@ jobs:
           path: dist/*
           retention-days: 7
 
+      - name: Save Bad Captures
+        if: failure() && steps.test.conclusion == 'failure'
+        uses: actions/upload-artifact@v3
+        with:
+          name: bad-captures
+          path: taichi-release-tests/bad-compare/*
+          retention-days: 7
+
   build_android_demos:
     name: Build Android Demos
     # Skip this job when testing the offline cache