From 505de43913bc678bcdf3bb568fdc6e263e9fa69e Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Tue, 27 Apr 2021 15:45:43 -0700 Subject: [PATCH] Parallelize test-lib and RPC tests --- .github/ci.sh | 10 --- .github/workflows/ci.yml | 102 +++++++++++++++++++++++++--- cryptol-remote-api/run_rpc_tests.sh | 50 +++++++------- 3 files changed, 115 insertions(+), 47 deletions(-) diff --git a/.github/ci.sh b/.github/ci.sh index ec826eeb3..74568b2f2 100755 --- a/.github/ci.sh +++ b/.github/ci.sh @@ -37,10 +37,6 @@ retry() { done } -setup_external_tools() { - cabal v2-install --install-method=copy --installdir="$BIN" test-lib -} - setup_dist_bins() { extract_exe "cryptol" "dist/bin" extract_exe "cryptol-html" "dist/bin" @@ -125,12 +121,6 @@ install_system_deps() { is_exe "$BIN" z3 && is_exe "$BIN" cvc4 && is_exe "$BIN" yices } -test_dist() { - setup_external_tools - echo "test-runner version: $($BIN/test-runner --version)" - $BIN/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol tests -} - check_docs() { ./cry build exe:check-exercises find ./docs/ProgrammingCryptol -name '*.tex' -print0 | xargs -0 -n1 cabal v2-exec check-exercises diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47be4f300..0f7d50291 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,11 +54,13 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - ghc: ["8.6.5", "8.8.4", "8.10.2"] + ghc-version: ["8.6.5", "8.8.4", "8.10.2"] exclude: # https://gitlab.haskell.org/ghc/ghc/-/issues/18550 - os: windows-latest - ghc: 8.10.2 + ghc-version: 8.10.2 + outputs: + test-lib-json: ${{ steps.test-lib.outputs.targets-json }} env: VERSION: ${{ needs.config.outputs.version }} RELEASE: ${{ needs.config.outputs.release }} @@ -71,10 +73,10 @@ jobs: with: python-version: '3.x' - - uses: actions/setup-haskell@v1 + - uses: haskell/actions/setup@v1 id: setup-haskell with: - ghc-version: ${{ matrix.ghc }} + ghc-version: ${{ matrix.ghc-version }} - uses: actions/cache@v2 name: Cache cabal store @@ -82,9 +84,9 @@ jobs: path: | ${{ steps.setup-haskell.outputs.cabal-store }} dist-newstyle - key: cabal-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles(format('cabal.GHC-{0}.config', matrix.ghc)) }}-${{ github.sha }} + key: cabal-${{ runner.os }}-${{ matrix.ghc-version }}-${{ hashFiles(format('cabal.GHC-{0}.config', matrix.ghc-version)) }}-${{ github.sha }} restore-keys: | - cabal-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles(format('cabal.GHC-{0}.config', matrix.ghc)) }}- + cabal-${{ runner.os }}-${{ matrix.ghc-version }}-${{ hashFiles(format('cabal.GHC-{0}.config', matrix.ghc-version)) }}- - shell: bash run: .github/ci.sh install_system_deps @@ -113,6 +115,20 @@ jobs: make " + - shell: bash + name: Partition test-lib tests + id: test-lib + run: | + set -x + cabal v2-install --install-method=copy --installdir="./bin" test-lib + cmd="cat \$1.stdout" + if ${{ runner.os == 'Windows' }}; then + cmd="cat \$1.stdout.mingw32 2>/dev/null || $cmd" + fi + ./bin/test-runner --ext=.icry -r ./output --exe=$(which bash) -F -c -F "$cmd" -F -- ./tests + TARGETS_JSON=$(echo -n "$(ls -1 ./output/tests)" | jq -Rsc 'split("\n")') + echo "::set-output name=targets-json::$TARGETS_JSON" + - shell: bash run: .github/ci.sh bundle_files @@ -146,6 +162,12 @@ jobs: if-no-files-found: error retention-days: ${{ needs.config.outputs.retention-days }} + - if: matrix.ghc-version == '8.6.5' + uses: actions/upload-artifact@v2 + with: + path: dist/bin + name: ${{ runner.os }}-bins + - uses: actions/upload-artifact@v2 if: runner.os == 'Windows' with: @@ -154,13 +176,71 @@ jobs: if-no-files-found: error retention-days: ${{ needs.config.outputs.retention-days }} - - shell: bash - run: .github/ci.sh test_dist + test: + runs-on: ${{ matrix.os }} + needs: [build] + strategy: + fail-fast: false + matrix: + suite: [test-lib] + target: ${{ fromJson(needs.build.outputs.test-lib-json) }} + os: [ubuntu-latest, macos-latest, windows-latest] + continue-on-error: [false] + include: + - suite: rpc + target: '' + os: ubuntu-latest + continue-on-error: false + - suite: rpc + target: '' + os: macos-latest + continue-on-error: false + - suite: rpc + target: '' + os: windows + continue-on-error: true # TODO: get Python client to work on Windows + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - uses: haskell/actions/setup@v1 + with: + ghc-version: '8.10.2' + + - uses: actions/download-artifact@v2 + with: + name: "${{ runner.os }}-bins" + path: dist/bin - # TODO: get Python client to work on Windows - shell: bash - run: .github/ci.sh test_rpc - if: runner.os != 'Windows' + run: | + set -x + chmod +x dist/bin/cryptol + .github/ci.sh install_system_deps + echo $PATH + z3 --version || true + ./bin/z3.exe --version || true + ghc_ver="$(ghc --numeric-version)" + cp cabal.GHC-"$ghc_ver".config cabal.project.freeze + cabal v2-update + + - if: matrix.suite == 'test-lib' + shell: bash + continue-on-error: ${{ matrix.continue-on-error }} + name: test-lib ${{ matrix.target }} + run: | + cabal v2-install -j --install-method=copy --installdir="./bin" test-lib + echo $PATH + which z3 + z3 --version + ls -al ./bin + PATH=$PWD/bin:$PWD/dist/bin:$PATH ./bin/test-runner --ext=.icry -F -b --exe=dist/bin/cryptol ./tests/${{ matrix.target }} + + - if: matrix.suite == 'rpc' + shell: bash + continue-on-error: ${{ matrix.continue-on-error }} + run: cryptol-remote-api/run_rpc_tests.sh build-push-image: runs-on: ubuntu-latest diff --git a/cryptol-remote-api/run_rpc_tests.sh b/cryptol-remote-api/run_rpc_tests.sh index d6e8bc0a2..11cee4d0c 100755 --- a/cryptol-remote-api/run_rpc_tests.sh +++ b/cryptol-remote-api/run_rpc_tests.sh @@ -2,11 +2,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -pushd $DIR - -cabal v2-build exe:cryptol-remote-api -cabal v2-build exe:cryptol-eval-server - pushd $DIR/python NUM_FAILS=0 @@ -23,32 +18,35 @@ if [ $? -ne 0 ]; then NUM_FAILS=$(($NUM_FAILS+1)) fi -export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-remote-api) -if [[ -x "$CRYPTOL_SERVER" ]]; then - echo "Running cryptol-remote-api tests..." - echo "Using server $CRYPTOL_SERVER" - python3 -m unittest discover tests/cryptol - if [ $? -ne 0 ]; then - NUM_FAILS=$(($NUM_FAILS+1)) +export CRYPTOL_SERVER=$(which cryptol-remote-api) +if [[ ! -x "$CRYPTOL_SERVER" ]]; then + export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-remote-api) + if [[ ! -x "$CRYPTOL_SERVER" ]]; then + echo "could not locate cryptol-remote-api executable" + exit 1 fi -else - echo "could not find the cryptol-remote-api via `cabal v2-exec which`" - NUM_FAILS=$(($NUM_FAILS+1)) +fi +echo "Running cryptol-remote-api tests..." +echo "Using server $CRYPTOL_SERVER" +python3 -m unittest discover tests/cryptol +if [ $? -ne 0 ]; then + NUM_FAILS=$(($NUM_FAILS+1)) fi -export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-eval-server) -if [[ -x "$CRYPTOL_SERVER" ]]; then - echo "Running cryptol-eval-server tests..." - echo "Using server $CRYPTOL_SERVER" - python3 -m unittest discover tests/cryptol_eval - if [ $? -ne 0 ]; then - NUM_FAILS=$(($NUM_FAILS+1)) +export CRYPTOL_SERVER=$(which cryptol-eval-server) +if [[ ! -x "$CRYPTOL_SERVER" ]]; then + export CRYPTOL_SERVER=$(cabal v2-exec which cryptol-eval-server) + if [[ ! -x "$CRYPTOL_SERVER" ]]; then + echo "could not locate cryptol-eval-server executable" + exit 1 fi -else - echo "could not find the cryptol-eval-server via `cabal v2-exec which`" - NUM_FAILS=$(($NUM_FAILS+1)) fi -popd +echo "Running cryptol-eval-server tests..." +echo "Using server $CRYPTOL_SERVER" +python3 -m unittest discover tests/cryptol_eval +if [ $? -ne 0 ]; then + NUM_FAILS=$(($NUM_FAILS+1)) +fi popd