From c19d76f9b85aff8b5ec876d62c01bc845868ddec Mon Sep 17 00:00:00 2001 From: Tim van der Meij Date: Sat, 1 Aug 2020 20:49:24 +0200 Subject: [PATCH 1/4] Use a `for...of` loop in the `specDone` handler in the test reporter Moreover, remove a left-over reference to `test.py` since that was ported to JavaScript a long time ago. --- test/unit/testreporter.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/unit/testreporter.js b/test/unit/testreporter.js index c74825ae241e5..ab517d6f7c7e9 100644 --- a/test/unit/testreporter.js +++ b/test/unit/testreporter.js @@ -59,10 +59,9 @@ var TestReporter = function (browser) { if (result.failedExpectations.length === 0) { sendResult("TEST-PASSED", result.description); } else { - var failedMessages = ""; - var items = result.failedExpectations; - for (var i = 0, ii = items.length; i < ii; i++) { - failedMessages += items[i].message + " "; + let failedMessages = ""; + for (const item of result.failedExpectations) { + failedMessages += `${item.message} `; } sendResult("TEST-UNEXPECTED-FAIL", result.description, failedMessages); } @@ -71,7 +70,7 @@ var TestReporter = function (browser) { this.suiteDone = function (result) {}; this.jasmineDone = function () { - // Give the test.py some time process any queued up requests + // Give the test runner some time process any queued requests. setTimeout(sendQuitRequest, 500); }; }; From 662ac5548fe030a21288adce0c2e764c4a078d56 Mon Sep 17 00:00:00 2001 From: Tim van der Meij Date: Sat, 1 Aug 2020 20:57:51 +0200 Subject: [PATCH 2/4] Log suite start failures in the test runner --- test/unit/testreporter.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/unit/testreporter.js b/test/unit/testreporter.js index ab517d6f7c7e9..41502d6d32d34 100644 --- a/test/unit/testreporter.js +++ b/test/unit/testreporter.js @@ -51,7 +51,21 @@ var TestReporter = function (browser) { sendInfo("Started tests for " + browser + "."); }; - this.suiteStarted = function (result) {}; + this.suiteStarted = function (result) { + // Normally suite starts don't have to be reported because the individual + // specs inside them are reported, but it can happen that the suite cannot + // start, for instance due to an uncaught exception in `beforeEach`. This + // is problematic because the specs inside the suite will never be found + // and run, so if we don't report the suite start failure here it would be + // ignored silently, leading to passing tests even though some did not run. + if (result.failedExpectations.length > 0) { + let failedMessages = ""; + for (const item of result.failedExpectations) { + failedMessages += `${item.message} `; + } + sendResult("TEST-UNEXPECTED-FAIL", result.description, failedMessages); + } + }; this.specStarted = function (result) {}; From b789a0e2160a4b59ff4afd500c5313398338b481 Mon Sep 17 00:00:00 2001 From: Tim van der Meij Date: Sat, 1 Aug 2020 21:06:42 +0200 Subject: [PATCH 3/4] Log the total number of tests and the random seed in the test runner This might make debugging intermittent failures a bit easier in the future because it allows us to spot unexpected differences in the number of tests being run and allows us to run the tests locally in the same order in case of intermittent failures. --- test/unit/testreporter.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/unit/testreporter.js b/test/unit/testreporter.js index 41502d6d32d34..64691c3b43711 100644 --- a/test/unit/testreporter.js +++ b/test/unit/testreporter.js @@ -48,7 +48,10 @@ var TestReporter = function (browser) { this.jasmineStarted = function (suiteInfo) { this.runnerStartTime = this.now(); - sendInfo("Started tests for " + browser + "."); + + const total = suiteInfo.totalSpecsDefined; + const seed = suiteInfo.order.seed; + sendInfo(`Started ${total} tests for ${browser} with seed ${seed}.`); }; this.suiteStarted = function (result) { From c53e403049f4cde265a751d171fb2b7adcc91c18 Mon Sep 17 00:00:00 2001 From: Tim van der Meij Date: Sat, 1 Aug 2020 21:11:32 +0200 Subject: [PATCH 4/4] Update Jasmine to version 3.6.1 This is possible now that most intermittent unit test failures have been resolved by other patches. There is only one remaining, but it's very rare and doesn't have to block this update anymore. --- package-lock.json | 91 ++++++++++++++++++++++++++++++++++------------- package.json | 2 +- 2 files changed, 68 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index ed08acb6363c1..e342ea84ecc52 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2598,6 +2598,22 @@ } } }, + "@mrmlnc/readdir-enhanced": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", + "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==", + "dev": true, + "requires": { + "call-me-maybe": "^1.0.1", + "glob-to-regexp": "^0.3.0" + } + }, + "@nodelib/fs.stat": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz", + "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==", + "dev": true + }, "@types/babel-types": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.4.tgz", @@ -3898,6 +3914,12 @@ "unset-value": "^1.0.0" } }, + "call-me-maybe": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.1.tgz", + "integrity": "sha1-JtII6onje1y95gJQoV8DHBak1ms=", + "dev": true + }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -6140,6 +6162,31 @@ "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", "dev": true }, + "fast-glob": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-2.2.7.tgz", + "integrity": "sha512-g1KuQwHOZAmOZMuBtHdxDtju+T2RT8jgCC9aANsbpdiDDTSnjgfuVsIBNKbUeJI3oKMRExcfNDtJl4OhbffMsw==", + "dev": true, + "requires": { + "@mrmlnc/readdir-enhanced": "^2.2.1", + "@nodelib/fs.stat": "^1.1.2", + "glob-parent": "^3.1.0", + "is-glob": "^4.0.0", + "merge2": "^1.2.3", + "micromatch": "^3.1.10" + }, + "dependencies": { + "is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "requires": { + "is-extglob": "^2.1.1" + } + } + } + }, "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -7072,6 +7119,12 @@ "path-dirname": "^1.0.0" } }, + "glob-to-regexp": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz", + "integrity": "sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs=", + "dev": true + }, "glob-watcher": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/glob-watcher/-/glob-watcher-5.0.3.tgz", @@ -8073,35 +8126,19 @@ } }, "jasmine": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-3.5.0.tgz", - "integrity": "sha512-DYypSryORqzsGoMazemIHUfMkXM7I7easFaxAvNM3Mr6Xz3Fy36TupTrAOxZWN8MVKEU5xECv22J4tUQf3uBzQ==", + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-3.6.1.tgz", + "integrity": "sha512-Jqp8P6ZWkTVFGmJwBK46p+kJNrZCdqkQ4GL+PGuBXZwK1fM4ST9BizkYgIwCFqYYqnTizAy6+XG2Ej5dFrej9Q==", "dev": true, "requires": { - "glob": "^7.1.4", - "jasmine-core": "~3.5.0" - }, - "dependencies": { - "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - } + "fast-glob": "^2.2.6", + "jasmine-core": "~3.6.0" } }, "jasmine-core": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-3.5.0.tgz", - "integrity": "sha512-nCeAiw37MIMA9w9IXso7bRaLl+c/ef3wnxsoSAlYrzS+Ot0zTG6nU8G/cIfGkqpkjX2wNaIW9RFG0TwIFnG6bA==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-3.6.0.tgz", + "integrity": "sha512-8uQYa7zJN8hq9z+g8z1bqCfdC8eoDAeVnM5sfqs7KHv9/ifoJ500m018fpFc7RDaO6SWCLCXwo/wPSNcdYTgcw==", "dev": true }, "js-stringify": { @@ -8677,6 +8714,12 @@ "readable-stream": "^2.0.1" } }, + "merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true + }, "micromatch": { "version": "3.1.10", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", diff --git a/package.json b/package.json index 86ae36fa30416..463a0bc17f19f 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "gulp-rename": "^1.4.0", "gulp-replace": "^1.0.0", "gulp-zip": "^4.2.0", - "jasmine": "~3.5.0", + "jasmine": "^3.6.1", "jsdoc": "^3.6.5", "jstransformer-markdown-it": "^2.1.0", "merge-stream": "^1.0.1",