From 4a46412437dbb60c6cf64f07a2cbfafa8db6c35f Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Tue, 27 Apr 2021 15:45:43 -0700 Subject: [PATCH] Parallelize test-lib and RPC tests --- .github/ci.sh | 10 ----- .github/workflows/ci.yml | 68 ++++++++++++++++++++++++++--- cryptol-remote-api/run_rpc_tests.sh | 50 ++++++++++----------- 3 files changed, 87 insertions(+), 41 deletions(-) diff --git a/.github/ci.sh b/.github/ci.sh index ec826eeb3..74568b2f2 100755 --- a/.github/ci.sh +++ b/.github/ci.sh @@ -37,10 +37,6 @@ retry() { done } -setup_external_tools() { - cabal v2-install --install-method=copy --installdir="$BIN" test-lib -} - setup_dist_bins() { extract_exe "cryptol" "dist/bin" extract_exe "cryptol-html" "dist/bin" @@ -125,12 +121,6 @@ install_system_deps() { is_exe "$BIN" z3 && is_exe "$BIN" cvc4 && is_exe "$BIN" yices } -test_dist() { - setup_external_tools - echo "test-runner version: $($BIN/test-runner --version)" - $BIN/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol tests -} - check_docs() { ./cry build exe:check-exercises find ./docs/ProgrammingCryptol -name '*.tex' -print0 | xargs -0 -n1 cabal v2-exec check-exercises diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47be4f300..4e579bcb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,8 @@ jobs: # https://gitlab.haskell.org/ghc/ghc/-/issues/18550 - os: windows-latest ghc: 8.10.2 + outputs: + test-lib-json: ${{ steps.test-lib.outputs.targets-json }} env: VERSION: ${{ needs.config.outputs.version }} RELEASE: ${{ needs.config.outputs.release }} @@ -71,7 +73,7 @@ jobs: with: python-version: '3.x' - - uses: actions/setup-haskell@v1 + - uses: haskell/actions/setup@v1 id: setup-haskell with: ghc-version: ${{ matrix.ghc }} @@ -113,6 +115,28 @@ jobs: make " + - shell: bash + name: Partition test-lib tests + id: test-lib + run: | + set -x + cabal v2-install --install-method=copy --installdir="./bin" test-lib + stdout=$(mktemp) + cat > $stdout << EndOfScript + if ${{ runner.os == 'Windows' }}; then + file="\$(cygpath -u "\$1").stdout" + else + file="\$1.stdout" + fi + echo "\$file" >> $PWD/lol + cat "\$file" + EndOfScript + ${{ runner.os == 'Windows' }} || chmod +x $stdout + ./bin/test-runner --ext=.icry -r ./output --exe=bash -F $stdout ./tests || true + cat ./lol + TARGETS_JSON=$(echo -n "$(ls -1 ./output/tests)" | jq -Rsc 'split("\n")') + echo "::set-output name=targets-json::$TARGETS_JSON" + - shell: bash run: .github/ci.sh bundle_files @@ -146,6 +170,12 @@ jobs: if-no-files-found: error retention-days: ${{ needs.config.outputs.retention-days }} + - if: matrix.ghc == '8.6.5' + uses: actions/upload-artifact@v2 + with: + path: dist/bin + name: ${{ runner.os }}-bins + - uses: actions/upload-artifact@v2 if: runner.os == 'Windows' with: @@ -154,13 +184,41 @@ jobs: if-no-files-found: error retention-days: ${{ needs.config.outputs.retention-days }} + test-lib: + runs-on: ${{ matrix.os }} + needs: [build] + strategy: + fail-fast: false + matrix: + target: ${{ fromJson(needs.build.outputs.test-lib-json) }} + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/download-artifact@v2 + with: + name: "${{ runner.os }}-bins" + path: dist/bin - shell: bash - run: .github/ci.sh test_dist + run: | + cabal v2-install --install-method=copy --installdir="./bin" test-lib + echo "test-runner version: $($BIN/test-runner --version)" + ./bin/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol ./tests/${{ matrix.target }} - # TODO: get Python client to work on Windows + test-rpc: + runs-on: ${{ matrix.os }} + needs: [build] + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + exclude: + - os: windows-latest # TODO: get Python client to work on Windows + steps: + - uses: actions/download-artifact@v2 + with: + name: "${{ runner.os }}-bins" + path: dist/bin - shell: bash - run: .github/ci.sh test_rpc - if: runner.os != 'Windows' + run: PATH=$PWD/dist/bin:$PATH cryptol-remote-api/run_rpc_tests.sh build-push-image: runs-on: ubuntu-latest diff --git a/cryptol-remote-api/run_rpc_tests.sh b/cryptol-remote-api/run_rpc_tests.sh index d6e8bc0a2..11cee4d0c 100755 --- a/cryptol-remote-api/run_rpc_tests.sh +++ b/cryptol-remote-api/run_rpc_tests.sh @@ -2,11 +2,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -pushd $DIR - -cabal v2-build exe:cryptol-remote-api -cabal v2-build exe:cryptol-eval-server - pushd $DIR/python NUM_FAILS=0 @@ -23,32 +18,35 @@ if [ $? -ne 0 ]; then NUM_FAILS=$(($NUM_FAILS+1)) fi -export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-remote-api) -if [[ -x "$CRYPTOL_SERVER" ]]; then - echo "Running cryptol-remote-api tests..." - echo "Using server $CRYPTOL_SERVER" - python3 -m unittest discover tests/cryptol - if [ $? -ne 0 ]; then - NUM_FAILS=$(($NUM_FAILS+1)) +export CRYPTOL_SERVER=$(which cryptol-remote-api) +if [[ ! -x "$CRYPTOL_SERVER" ]]; then + export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-remote-api) + if [[ ! -x "$CRYPTOL_SERVER" ]]; then + echo "could not locate cryptol-remote-api executable" + exit 1 fi -else - echo "could not find the cryptol-remote-api via `cabal v2-exec which`" - NUM_FAILS=$(($NUM_FAILS+1)) +fi +echo "Running cryptol-remote-api tests..." +echo "Using server $CRYPTOL_SERVER" +python3 -m unittest discover tests/cryptol +if [ $? -ne 0 ]; then + NUM_FAILS=$(($NUM_FAILS+1)) fi -export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-eval-server) -if [[ -x "$CRYPTOL_SERVER" ]]; then - echo "Running cryptol-eval-server tests..." - echo "Using server $CRYPTOL_SERVER" - python3 -m unittest discover tests/cryptol_eval - if [ $? -ne 0 ]; then - NUM_FAILS=$(($NUM_FAILS+1)) +export CRYPTOL_SERVER=$(which cryptol-eval-server) +if [[ ! -x "$CRYPTOL_SERVER" ]]; then + export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-eval-server) + if [[ ! -x "$CRYPTOL_SERVER" ]]; then + echo "could not locate cryptol-eval-server executable" + exit 1 fi -else - echo "could not find the cryptol-eval-server via `cabal v2-exec which`" - NUM_FAILS=$(($NUM_FAILS+1)) fi -popd +echo "Running cryptol-eval-server tests..." +echo "Using server $CRYPTOL_SERVER" +python3 -m unittest discover tests/cryptol_eval +if [ $? -ne 0 ]; then + NUM_FAILS=$(($NUM_FAILS+1)) +fi popd