Tweak smoke tests (#62)

.dockerignore

@@ -9,3 +9,7 @@ dev.Dockerfile
 .gitignore
 .gitattributes
 .dockerignore
+/bin/
+!/bin/run.sh
+!/bin/prepare.sh
+test/

.github/workflows/ci.js.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install project dependencies
         run: yarn install --frozen-lockfile
 
-      - name: Run exercism/javascript-test-runner ci precheck (lint code)
+      - name: Run exercism/typescript-test-runner ci precheck (lint code)
         run: bin/lint.sh
 
   ci:
@@ -43,3 +43,21 @@ jobs:
 
       - name: Build the test-runner (using Node ${{ matrix.node-version }})
         run: bin/test.sh
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1
+        with:
+          install: true
+
+      - name: Build Docker image and store in cache
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825
+        with:
+          context: .
+          push: false
+          load: true
+          tags: exercism/typescript-test-runner
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Tests in Docker
+        run: bin/run-tests-in-docker.sh

.github/workflows/pr.ci.js.yml

@@ -45,3 +45,21 @@ jobs:
 
       - name: Build the test-runner (using Node ${{ matrix.node-version }})
         run: bin/test.sh
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1
+        with:
+          install: true
+
+      - name: Build Docker image and store in cache
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825
+        with:
+          context: .
+          push: false
+          load: true
+          tags: exercism/typescript-test-runner
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Tests in Docker
+        run: bin/run-tests-in-docker.sh

bin/run-tests-in-docker.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the test runner Docker image by running it against a predefined set of 
+# solutions with an expected output.
+# The test runner Docker image is built automatically.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner Docker image.
+
+# Example:
+# ./bin/run-tests-in-docker.sh
+
+# Stop executing when a command returns a non-zero return code
+set -e
+
+# Build the Docker image
+docker build --rm -t exercism/typescript-test-runner .
+
+# Run the Docker image using the settings mimicking the production environment
+docker run \
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src="${PWD}/test/fixtures",dst=/opt/test-runner/test/fixtures \
+    --mount type=tmpfs,dst=/tmp \
+    --volume "${PWD}/bin/run-tests.sh:/opt/test-runner/bin/run-tests.sh" \
+    --workdir /opt/test-runner \
+    --entrypoint /opt/test-runner/bin/run-tests.sh \
+    exercism/typescript-test-runner

bin/run-tests.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# Synopsis:
+# Test the test runner by running it against a predefined set of solutions 
+# with an expected output.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner.
+
+# Example:
+# ./bin/run-tests.sh
+
+exit_code=0
+
+# We need to copy the fixtures to a temp directory as the user
+# running within the Docker container does not have permissions 
+# to run the sed command on the fixtures directory
+fixtures_dir="test/fixtures"
+tmp_fixtures_dir="/tmp/test/fixtures"
+rm -rf "${tmp_fixtures_dir}"
+mkdir -p "${tmp_fixtures_dir}"
+cp -R ${fixtures_dir}/* "${tmp_fixtures_dir}"
+
+# Iterate over all test directories
+for test_file in $(find "${tmp_fixtures_dir}" -name '*.test.ts'); do
+    slug=$(echo "${test_file:${#tmp_fixtures_dir}+1}" | cut -d / -f 1)
+    test_dir=$(dirname "${test_file}")
+    test_dir_name=$(basename "${test_dir}")
+    test_dir_path=$(realpath "${test_dir}")
+    results_file_path="${test_dir_path}/results.json"
+    expected_results_file_path="${test_dir_path}/expected_results.json"
+
+    # Make sure there is no existing node_modules directory
+    rm -rf "${test_dir_path}/node_modules"
+
+    bin/run.sh "${slug}" "${test_dir_path}" "${test_dir_path}"
+
+    echo "${slug}/${test_dir_name}: comparing results.json to expected_results.json"
+    diff "${results_file_path}" "${expected_results_file_path}"
+
+    if [ $? -ne 0 ]; then
+        exit_code=1
+    fi
+done
+
+exit ${exit_code}

test/fixtures/pythagorean-triplet/chroth7/expected_results.json

@@ -0,0 +1,69 @@
+{
+  "status": "pass",
+  "tests": [
+    {
+      "name": "Triplet > triplets whose sum is 12",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(12)).toEqual([[3, 4, 5]])"
+    },
+    {
+      "name": "Triplet > triplets whose sum is 108",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(108)).toEqual([[27, 36, 45]])"
+    },
+    {
+      "name": "Triplet > triplets whose sum is 1000",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(1000)).toEqual([[200, 375, 425]])"
+    },
+    {
+      "name": "Triplet > no matching triplets for 1001",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(1001)).toEqual([])"
+    },
+    {
+      "name": "Triplet > returns all matching triplets",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(90)).toEqual([\n      [9, 40, 41],\n      [15, 36, 39],\n    ])"
+    },
+    {
+      "name": "Triplet > several matching triplets",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(840)).toEqual([\n      [40, 399, 401],\n      [56, 390, 394],\n      [105, 360, 375],\n      [120, 350, 370],\n      [140, 336, 364],\n      [168, 315, 357],\n      [210, 280, 350],\n      [240, 252, 348],\n    ])"
+    },
+    {
+      "name": "Triplet > returns triplets with no factor smaller than minimum factor",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(90, { minFactor: 10 })).toEqual([[15, 36, 39]])"
+    },
+    {
+      "name": "Triplet > returns triplets with no factor larger than maximum factor",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(840, { maxFactor: 349 })).toEqual([[240, 252, 348]])"
+    },
+    {
+      "name": "Triplet > returns triplets with factors in range",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "expect(tripletsWithSum(840, { maxFactor: 352, minFactor: 150 })).toEqual([\n      [210, 280, 350],\n      [240, 252, 348],\n    ])"
+    }
+  ],
+  "version": 3
+}

test/fixtures/two-fer/error/malformed_tests/expected_results.json

@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "message": "SyntaxError: <solution>/two-fer.test.ts: Unexpected token (2:0)\n\n    \u001b[0m \u001b[90m 1 |\u001b[39m describe(\u001b[32m'twoFer()'\u001b[39m\u001b[33m,\u001b[39m t(\u001b[32m'another name given'\u001b[39m\u001b[33m,\u001b[39m () \u001b[33m=>\u001b[39m {\u001b[0m\n    \u001b[0m\u001b[31m\u001b[1m>\u001b[22m\u001b[39m\u001b[90m 2 |\u001b[39m\u001b[0m\n    \u001b[0m \u001b[90m   |\u001b[39m \u001b[31m\u001b[1m^\u001b[22m\u001b[39m\u001b[0m\n",
+  "tests": [],
+  "version": 3
+}

test/fixtures/two-fer/error/syntax/expected_results.json

@@ -0,0 +1 @@
+{ "version": 1, "status": "error", "message": "The submitted code didn't compile. We have collected the errors encountered during compilation. At this moment the error messages are not very read-friendly, but it's a start. We are working on a more helpful output.\n-------------------------------\ntwo-fer.ts(1,14): error TS1389: 'const' is not allowed as a variable declaration name.\ntwo-fer.ts(1,20): error TS1134: Variable declaration expected.\ntwo-fer.ts(1,29): error TS1109: Expression expected.\ntwo-fer.ts(2,1): error TS1005: ')' expected." }

test/fixtures/two-fer/fail/empty/expected_results.json

@@ -0,0 +1,27 @@
+{
+  "status": "fail",
+  "tests": [
+    {
+      "name": "TwoFer > no name given",
+      "status": "fail",
+      "message": "TypeError: (0 , _twoFer.twoFer) is not a function",
+      "output": null,
+      "test_code": "const expected = 'One for you, one for me.'\nexpect(twoFer()).toEqual(expected)"
+    },
+    {
+      "name": "TwoFer > a name given",
+      "status": "fail",
+      "message": "TypeError: (0 , _twoFer.twoFer) is not a function",
+      "output": null,
+      "test_code": "const expected = 'One for Alice, one for me.'\nexpect(twoFer('Alice')).toEqual(expected)"
+    },
+    {
+      "name": "TwoFer > another name given",
+      "status": "fail",
+      "message": "TypeError: (0 , _twoFer.twoFer) is not a function",
+      "output": null,
+      "test_code": "const expected = 'One for Bob, one for me.'\nexpect(twoFer('Bob')).toEqual(expected)"
+    }
+  ],
+  "version": 3
+}

test/fixtures/two-fer/fail/tests/expected_results.json

@@ -0,0 +1,27 @@
+{
+  "status": "fail",
+  "tests": [
+    {
+      "name": "TwoFer > no name given",
+      "status": "pass",
+      "message": "",
+      "output": "[log] ",
+      "test_code": "const expected = 'One for you, one for me.'\nexpect(twoFer('')).toEqual(expected)"
+    },
+    {
+      "name": "TwoFer > a name given",
+      "status": "fail",
+      "message": "Error: \u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).\u001b[22mtoEqual\u001b[2m(\u001b[22m\u001b[32mexpected\u001b[39m\u001b[2m) // deep equality\u001b[22m\n\nExpected: \u001b[32m\"One for \u001b[7mAlice\u001b[27m, one for me.\"\u001b[39m\nReceived: \u001b[31m\"One for \u001b[7myou\u001b[27m, one for me.\"\u001b[39m",
+      "output": "[log] Some global log\n[log] Alice",
+      "test_code": "const expected = 'One for Alice, one for me.'\nexpect(twoFer('Alice')).toEqual(expected)"
+    },
+    {
+      "name": "TwoFer > another name given",
+      "status": "fail",
+      "message": "Error: \u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).\u001b[22mtoEqual\u001b[2m(\u001b[22m\u001b[32mexpected\u001b[39m\u001b[2m) // deep equality\u001b[22m\n\nExpected: \u001b[32m\"One for \u001b[7mBob\u001b[27m, one for me.\"\u001b[39m\nReceived: \u001b[31m\"One for \u001b[7myou\u001b[27m, one for me.\"\u001b[39m",
+      "output": "[log] Bob",
+      "test_code": "const expected = 'One for Bob, one for me.'\nexpect(twoFer('Bob')).toEqual(expected)"
+    }
+  ],
+  "version": 3
+}

test/fixtures/two-fer/pass/expected_results.json

@@ -0,0 +1,27 @@
+{
+  "status": "pass",
+  "tests": [
+    {
+      "name": "TwoFer > no name given",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "const expected = 'One for you, one for me.'\nexpect(twoFer()).toEqual(expected)"
+    },
+    {
+      "name": "TwoFer > a name given",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "const expected = 'One for Alice, one for me.'\nexpect(twoFer('Alice')).toEqual(expected)"
+    },
+    {
+      "name": "TwoFer > another name given",
+      "status": "pass",
+      "message": "",
+      "output": null,
+      "test_code": "const expected = 'One for Bob, one for me.'\nexpect(twoFer('Bob')).toEqual(expected)"
+    }
+  ],
+  "version": 3
+}