CI: Run tests using julia-test-runner (JuliaLang#456)

.github/workflows/exercise-tests.yml

@@ -26,14 +26,14 @@ jobs:
         os: [ubuntu-latest, windows-latest, macOS-latest]
 
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
 
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@ee66464cb7897ffcc5322800f4b18d449794af30
         with:
           version: ${{ matrix.julia-version }}
 
       - name: Cache Julia artifacts
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
         with:
           path: ~/.julia/artifacts 
           key: ${{ runner.os }}-${{ matrix.julia-version }}-test-cache-artifacts-${{ hashFiles('**/Project.toml') }}
@@ -47,3 +47,46 @@ jobs:
 
       - name: Test exercises
         run: julia --color=yes --project runtests.jl
+
+  test-runner:
+    name: Julia Test Runner
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+
+      - name: Pull julia-test-runner image
+        run: docker pull exercism/julia-test-runner
+
+      - name: Install Julia dependencies
+        run: julia --color=yes --project -e "using Pkg; Pkg.instantiate()"
+
+      - name: Generate test reports using julia-test-runner
+        id: generate-reports
+        run: julia --color=yes --project runtestrunner.jl
+
+      - name: Upload reports as artifact
+        uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074
+        with:
+          name: test-reports
+          path: ${{ steps.generate-reports.outputs.results-path }}
+
+      - name: Check if all tests passed
+        run: |
+          using JSON
+
+          any_errors = false
+
+          for report in readdir(ENV["RESULTS"])
+            result = JSON.parsefile(joinpath(ENV["RESULTS"], report))
+            if result["status"] != "pass"
+              global any_errors = true
+              @error "Exercise failed tests" report
+              run(`jq -C '.' $(joinpath(ENV["RESULTS"], report))`)
+            end
+          end
+
+          any_errors && exit(1)
+        env:
+          RESULTS: ${{ steps.generate-reports.outputs.results-path }}
+        shell: julia --color=yes --project {0}

runtestrunner.jl

@@ -0,0 +1,50 @@
+using JSON
+
+# Setup GHA logger in CI
+using Logging: global_logger
+using GitHubActions: GitHubActionsLogger, set_output
+get(ENV, "GITHUB_ACTIONS", "false") == "true" && global_logger(GitHubActionsLogger())
+
+# Dir to store generated test results in
+results_dir = mktempdir(; prefix="jl_test-runner_results_", cleanup=false)
+
+# In CI, set results directory as output
+set_output("results-path", results_dir)
+
+for exercise_type in ("concept", "practice")
+    for exercise in readdir(joinpath("exercises", exercise_type))
+        # Allow only testing specified exercises
+        if !isempty(ARGS) && !(exercise in ARGS)
+            continue
+        end
+
+        exercise_path = joinpath("exercises", exercise_type, exercise)
+        isdir(exercise_path) || continue
+
+        # Concept exercises have an exemplar, practice exercises merely an example for CI purposes
+        example_or_exemplar = exercise_type == "concept" ? "exemplar" : "example"
+
+        @info "[$(uppercase(exercise_type))] Testing $exercise"
+
+        # Determine file name of the solution
+        meta_cfg = JSON.parsefile(joinpath(exercise_path, ".meta", "config.json"))
+        solution_file = meta_cfg["files"]["solution"][1]
+
+        # Copy solution to temporary directory
+        tmp = mktempdir(; prefix="jl_$(exercise)_")
+        cp(joinpath(exercise_path, ".meta", example_or_exemplar * ".jl"), joinpath(tmp, solution_file))
+        cp(joinpath(exercise_path, "runtests.jl"), joinpath(tmp, "runtests.jl"))
+
+        # Run test-runner
+        run(`docker run
+                --network none
+                --read-only
+                --mount type=bind,src=$tmp,dst=/solution
+                --mount type=bind,src=$tmp,dst=/output
+                --mount type=tmpfs,dst=/tmp
+                exercism/julia-test-runner $exercise /solution /output`
+        )
+
+        cp(joinpath(tmp, "results.json"), joinpath(results_dir, "$(exercise_type)_$exercise.json"))
+    end
+end