Create test runner

.gitignore

@@ -1 +1,8 @@
-tests/*/results.json
+**/*/results.json
+**/*/*.beam
+**/*/*.ez
+**/*/build
+**/*/erl_crash.dump
+**/*/.gitignore
+**/*/.github
+**/*/README.md

Dockerfile

@@ -1,8 +1,13 @@
-FROM alpine:3.10
+FROM ghcr.io/gleam-lang/gleam:v0.24.0-erlang-alpine
 
-# install packages required to run the tests
+# Install packages required to run the tests
 RUN apk add --no-cache jq coreutils
 
 WORKDIR /opt/test-runner
 COPY . .
+
+# Download the used Gleam packages eagerly as the test runner will not have
+# network access to do so.
+RUN cd packages && gleam deps download
+
 ENTRYPOINT ["/opt/test-runner/bin/run.sh"]

README.md

@@ -1,21 +1,4 @@
-# Exercism Test Runner Template
-
-This repository is a [template repository](https://help.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-template-repository) for creating [test runners][test-runners] for [Exercism][exercism] tracks.
-
-## Using the Test Runner Template
-
-1. Ensure that your track has not already implemented a test runner. If there is, there will be a `https://github.com/exercism/<track>-test-runner` repository (i.e. if your track's slug is `python`, the test runner repo would be `https://github.com/exercism/python-test-runner`)
-2. Follow [GitHub's documentation](https://help.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-repository-from-a-template) for creating a repository from a template repository
-   - Name your new repository based on your language track's slug (i.e. if your track is for Python, your test runner repo name is `python-test-runner`)
-3. Remove this [Exercism Test Runner Template](#exercism-test-runner-template) section from the `README.md` file
-4. Build the test runner, conforming to the [Test Runner interface specification](https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md).
-   - Update the files to match your track's needs. At the very least, you'll need to update `bin/run.sh`, `Dockerfile` and the test solutions in the `tests` directory
-   - Tip: look for `TODO:` comments to point you towards code that need updating
-   - Tip: look for `OPTIONAL:` comments to point you towards code that _could_ be useful
-
-Once you're happy with your test runner, [open an issue on the exercism/exercism](https://github.com/exercism/exercism/issues/new?assignees=&labels=&template=new-test-runner.md&title=%5BNew+Test+Runner%5D+) to request an official test runner repository for your track.
-
-# Exercism TRACK_NAME_HERE Test Runner
+# Exercism Gleam Test Runner
 
 The Docker image to automatically run tests on TRACK_NAME_HERE solutions submitted to [Exercism].
 
@@ -44,7 +27,8 @@ Once the test runner has finished, its results will be written to `<output-dir>/
 To run the tests to verify the behavior of the test runner, do the following:
 
 1. Open a terminal in the project's root
-2. Run `./bin/run-tests.sh`
+2. Run `./bin/setup-locally.sh`
+3. Run `./bin/run-tests.sh`
 
 These are [golden tests][golden] that compare the `results.json` generated by running the current state of the code against the "known good" `tests/<test-name>/results.json`. All files created during the test run itself are discarded.
 

bin/run-tests.sh

@@ -11,32 +11,37 @@
 # Example:
 # ./bin/run-tests.sh
 
+set -eu
+
 exit_code=0
 
 # Iterate over all test directories
 for test_dir in tests/*; do
-    test_dir_name=$(basename "${test_dir}")
-    test_dir_path=$(realpath "${test_dir}")
-    results_file_path="${test_dir_path}/results.json"
-    expected_results_file_path="${test_dir_path}/expected_results.json"
-
-    bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
-
-    # OPTIONAL: Normalize the results file
-    # If the results.json file contains information that changes between 
-    # different test runs (e.g. timing information or paths), you should normalize
-    # the results file to allow the diff comparison below to work as expected
-    # sed -i -E \
-    #   -e 's/Elapsed time: [0-9]+\.[0-9]+ seconds//g' \
-    #   -e "s~${test_dir_path}~/solution~g" \
-    #   "${results_file_path}"
-
-    echo "${test_dir_name}: comparing results.json to expected_results.json"
-    diff "${results_file_path}" "${expected_results_file_path}"
-
-    if [ $? -ne 0 ]; then
-        exit_code=1
-    fi
+  test_dir_name=$(basename "${test_dir}")
+  test_dir_path=$(realpath "${test_dir}")
+  results_file_path="${test_dir_path}/results.json"
+  expected_results_file_path="${test_dir_path}/expected_results.json"
+
+  echo "${test_dir_name}: testing..."
+  bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}" > /dev/null
+
+  if diff "${results_file_path}" "${expected_results_file_path}"
+  then
+    echo "${test_dir_name}: pass"
+  else
+    echo
+    echo "${test_dir_name}: fail."
+    echo "${test_dir_name}: ${results_file_path} does not match ${expected_results_file_path}"
+    exit_code=1
+  fi
+  echo
 done
 
+if [ "${exit_code}" -eq 0 ]
+then
+  echo "All tests passed!"
+else
+  echo "Some tests failed!"
+fi
+
 exit ${exit_code}

bin/run.sh

@@ -17,44 +17,66 @@
 
 # If any required arguments is missing, print the usage and exit
 if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
-    echo "usage: ./bin/run.sh exercise-slug path/to/solution/folder/ path/to/output/directory/"
-    exit 1
+  echo "usage: ./bin/run.sh exercise-slug path/to/solution/folder/ path/to/output/directory/"
+  exit 1
 fi
 
+set -eu
+
+root_dir=$(dirname "$(dirname "$(realpath "$0")")")
 slug="$1"
 solution_dir=$(realpath "${2%/}")
 output_dir=$(realpath "${3%/}")
 results_file="${output_dir}/results.json"
 
+echo "Copying dependencies..."
+cd packages
+mkdir -p "$solution_dir"/build
+cp -r "$root_dir"/packages/build/packages "$solution_dir"/build/packages
+
+sanitise_gleam_output() {
+  grep -vE \
+    -e "^Downloading packages" \
+    -e "^ Downloaded [0-9]+ packages in [0-9]\.[0-9]+s" \
+    -e "^ Downloaded [0-9]+ packages in [0-9]\.[0-9]+s" \
+    -e "^  Compiling [a-z0-9_]+$" \
+    -e "^   Compiled in [0-9]+\.[0-9]+s" \
+    -e "^    Running [a-z0-9_]+\.main" \
+    -e "^Finished in [0-9]+\.[0-9]+"
+}
+
+
 # Create the output directory if it doesn't exist
 mkdir -p "${output_dir}"
 
+# Compile the project
+echo "${slug}: compiling..."
+
+cd "${solution_dir}" || exit 1
+
+# Reset the build directory
+gleam clean
+
+if ! output=$(gleam build 2>&1)
+then
+  output=$(echo "${output}" | sanitise_gleam_output)
+  jq -n --arg output "${output}" '{version: 1, status: "error", message: $output}' > "${results_file}"
+  echo "Compilation contained error, see ${output_dir}/results.json"
+  exit 0
+fi
+
 echo "${slug}: testing..."
 
 # Run the tests for the provided implementation file and redirect stdout and
 # stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
-
 # Write the results.json file based on the exit code of the command that was 
 # just executed that tested the implementation file
-if [ $? -eq 0 ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
+if output=$(gleam test 2>&1)
+then
+  jq -n '{version: 1, status: "pass"}' > "${results_file}"
 else
-    # OPTIONAL: Sanitize the output
-    # In some cases, the test output might be overly verbose, in which case stripping
-    # the unneeded information can be very helpful to the student
-    # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
-
-    # OPTIONAL: Manually add colors to the output to help scanning the output for errors
-    # If the test output does not contain colors to help identify failing (or passing)
-    # tests, it can be helpful to manually add colors to the output
-    # colorized_test_output=$(echo "${test_output}" \
-    #      | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
-    #      | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
-
-    jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
+  output=$(echo "${output}" | sanitise_gleam_output)
+  jq -n --arg output "${output}" '{version: 1, status: "fail", message: $output}' > "${results_file}"
 fi
 
 echo "${slug}: done"

bin/setup-locally.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Setup the test runner locally by pre-downloading dependency packages.
+
+# Example:
+# ./bin/setup-locally.sh
+
+set -eu
+
+root_dir=$(dirname "$(dirname "$(realpath "$0")")")
+cd "$root_dir"/packages
+gleam deps download

packages/gleam.toml

@@ -0,0 +1,16 @@
+name = "packages"
+version = "0.1.0"
+
+# Fill out these fields if you intend to generate HTML documentation or publish
+# your project to the Hex package manager.
+#
+# licences = ["Apache-2.0"]
+# description = "A Gleam library..."
+# repository = { type = "github", user = "username", repo = "project" }
+# links = [{ title = "Website", href = "https://gleam.run" }]
+
+[dependencies]
+gleam_stdlib = "~> 0.24"
+
+[dev-dependencies]
+gleeunit = "~> 0.6"

packages/manifest.toml

@@ -0,0 +1,11 @@
+# This file was generated by Gleam
+# You typically do not need to edit this file
+
+packages = [
+  { name = "gleam_stdlib", version = "0.24.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "BD0E75F3153375218E9EC380930E3621036CADCF4246AA55B193BDABFD06D5CE" },
+  { name = "gleeunit", version = "0.7.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "6F88395CAC9363EBE3D632E62C2C6EF731BBFB0A8D31E5E7287BC400002F15B2" },
+]
+
+[requirements]
+gleam_stdlib = "~> 0.24"
+gleeunit = "~> 0.6"

packages/src/packages.gleam

@@ -0,0 +1,5 @@
+import gleam/io
+
+pub fn main() {
+  io.println("Hello from packages!")
+}

tests/all_fail/expected_results.json

@@ -0,0 +1,5 @@
+{
+  "version": 1,
+  "status": "fail",
+  "message": "\u001b[0;31mF\u001b[0m\u001b[0;31mF\u001b[0m\nFailures:\n\n  1) all_fail_test:one_test/0\n\u001b[0;31m     Failure: ?assertEqual(<<111,111,112,115>>, Actual)\n       expected: <<\"oops\">>\n            got: <<\"one\">>\u001b[0m\n\u001b[0;36m     %% eunit_proc.erl:346:in `eunit_proc:with_timeout/3`\u001b[0m\n\u001b[0;36m     Output: \u001b[0m\n\u001b[0;36m     Output: \u001b[0m\n\n  2) all_fail_test:two_test/0\n\u001b[0;31m     Failure: ?assertEqual(<<111,111,112,115>>, Actual)\n       expected: <<\"oops\">>\n            got: <<\"two\">>\u001b[0m\n\u001b[0;36m     %% eunit_proc.erl:346:in `eunit_proc:with_timeout/3`\u001b[0m\n\u001b[0;36m     Output: \u001b[0m\n\u001b[0;36m     Output: \u001b[0m\n\n\u001b[0;31m2 tests, 2 failures\n\u001b[0m"
+}

tests/all_fail/gleam.toml

@@ -0,0 +1,16 @@
+name = "all_fail"
+version = "0.1.0"
+
+# Fill out these fields if you intend to generate HTML documentation or publish
+# your project to the Hex package manager.
+#
+# licences = ["Apache-2.0"]
+# description = "A Gleam library..."
+# repository = { type = "github", user = "username", repo = "project" }
+# links = [{ title = "Website", href = "https://gleam.run" }]
+
+[dependencies]
+gleam_stdlib = "~> 0.24"
+
+[dev-dependencies]
+gleeunit = "~> 0.6"

tests/all_fail/manifest.toml

@@ -0,0 +1,11 @@
+# This file was generated by Gleam
+# You typically do not need to edit this file
+
+packages = [
+  { name = "gleam_stdlib", version = "0.24.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "BD0E75F3153375218E9EC380930E3621036CADCF4246AA55B193BDABFD06D5CE" },
+  { name = "gleeunit", version = "0.7.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "6F88395CAC9363EBE3D632E62C2C6EF731BBFB0A8D31E5E7287BC400002F15B2" },
+]
+
+[requirements]
+gleam_stdlib = "~> 0.24"
+gleeunit = "~> 0.6"

tests/all_fail/src/all_fail.gleam

@@ -0,0 +1,5 @@
+import gleam/io
+
+pub fn main() {
+  io.println("Hello from all_fail!")
+}

tests/all_fail/test/all_fail_test.gleam

@@ -0,0 +1,16 @@
+import gleeunit
+import gleeunit/should
+
+pub fn main() {
+  gleeunit.main()
+}
+
+pub fn one_test() {
+  "one"
+  |> should.equal("oops")
+}
+
+pub fn two_test() {
+  "two"
+  |> should.equal("oops")
+}

tests/empty_file/expected_results.json

@@ -0,0 +1,5 @@
+{
+  "version": 1,
+  "status": "error",
+  "message": "error: Unknown module field\n   ┌─ ./test/empty_file_test.gleam:10:13\n   │\n10 │   empty_file.hello_world()\n   │             ^^^^^^^^^^^^\n\nThe module `empty_file` does not have a `hello_world` field."
+}

tests/empty_file/gleam.toml

@@ -0,0 +1,16 @@
+name = "empty_file"
+version = "0.1.0"
+
+# Fill out these fields if you intend to generate HTML documentation or publish
+# your project to the Hex package manager.
+#
+# licences = ["Apache-2.0"]
+# description = "A Gleam library..."
+# repository = { type = "github", user = "username", repo = "project" }
+# links = [{ title = "Website", href = "https://gleam.run" }]
+
+[dependencies]
+gleam_stdlib = "~> 0.24"
+
+[dev-dependencies]
+gleeunit = "~> 0.6"

tests/empty_file/manifest.toml

@@ -0,0 +1,11 @@
+# This file was generated by Gleam
+# You typically do not need to edit this file
+
+packages = [
+  { name = "gleam_stdlib", version = "0.24.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "BD0E75F3153375218E9EC380930E3621036CADCF4246AA55B193BDABFD06D5CE" },
+  { name = "gleeunit", version = "0.7.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "6F88395CAC9363EBE3D632E62C2C6EF731BBFB0A8D31E5E7287BC400002F15B2" },
+]
+
+[requirements]
+gleam_stdlib = "~> 0.24"
+gleeunit = "~> 0.6"

tests/empty_file/src/empty_file.gleam

@@ -0,0 +1 @@
+

tests/empty_file/test/empty_file_test.gleam

@@ -0,0 +1,12 @@
+import gleeunit
+import gleeunit/should
+import empty_file
+
+pub fn main() {
+  gleeunit.main()
+}
+
+pub fn hello_world_test() {
+  empty_file.hello_world()
+  |> should.equal("Hello, from empty_file!")
+}

tests/partial_fail/expected_results.json

@@ -0,0 +1,5 @@
+{
+  "version": 1,
+  "status": "fail",
+  "message": "\u001b[0;32m.\u001b[0m\u001b[0;32m.\u001b[0m\u001b[0;31mF\u001b[0m\nFailures:\n\n  1) partial_fail_test:three_test/0\n\u001b[0;31m     Failure: ?assertEqual(<<111,111,112,115>>, Actual)\n       expected: <<\"oops\">>\n            got: <<\"three\">>\u001b[0m\n\u001b[0;36m     %% eunit_proc.erl:346:in `eunit_proc:with_timeout/3`\u001b[0m\n\u001b[0;36m     Output: \u001b[0m\n\u001b[0;36m     Output: \u001b[0m\n\n\u001b[0;31m3 tests, 1 failures\n\u001b[0m"
+}