Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Test Adapter Improvements - Register test results incrementally #1430

Closed
wants to merge 13 commits into from
Closed

Test Adapter Improvements - Register test results incrementally #1430

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
8278208
return and record test results incrementally
ozyx Nov 2, 2016
49b050a
fix formatting
ozyx Nov 2, 2016
5baa46d
automatically fail tests that fail in before hooks
ozyx Nov 3, 2016
56ca7c5
separate test suites by testFile instead of workingDir
ozyx Nov 7, 2016
dbd2605
Fix newlines in TRX Test Result output
ozyx Nov 8, 2016
16f0bc5
modify exportrunner.js to return results incrementally
ozyx Nov 17, 2016
1efc985
modify mocha.js to return results incrementally
ozyx Nov 17, 2016
7349067
clean up code
ozyx Dec 3, 2016
a0acb7a
display test stdout and stderr in exportRunner test results
ozyx Dec 3, 2016
ef497da
code cleanup
ozyx Dec 5, 2016
bf150b7
modify tape.run_tests to be compatible with new TestExecutor
ozyx Dec 5, 2016
c5d1310
clean up code
ozyx Dec 6, 2016
b189e21
clean up code
ozyx Dec 6, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 31 additions & 16 deletions Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ var result = {
'title': '',
'passed': false,
'stdOut': '',
'stdErr': '',
'time': 0
'stdErr': ''
};

function append_stdout(string, encoding, fd) {
Expand All @@ -15,8 +14,13 @@ function append_stdout(string, encoding, fd) {
function append_stderr(string, encoding, fd) {
result.stdErr += string;
}
process.stdout.write = append_stdout;
process.stderr.write = append_stderr;
function hook_outputs() {
process.stdout.write = append_stdout;
process.stderr.write = append_stderr;
}


hook_outputs();

var find_tests = function (testFileList, discoverResultFile) {
var debug;
Expand Down Expand Up @@ -67,30 +71,41 @@ var find_tests = function (testFileList, discoverResultFile) {
module.exports.find_tests = find_tests;

var run_tests = function (testCases, callback) {
var test_results = [];
for (var test in testCases) {
function post(event) {
callback(event);
hook_outputs();
}

for (var test of testCases) {
post({
type: 'test start',
title: test.testName
});
try {
var testCase = require(testCases[test].testFile);
result.title = testCases[test].testName;
result.time = Date.now();
testCase[testCases[test].testName]();
result.time = Date.now() - result.time;
var testCase = require(test.testFile);
testCase[test.testName]();
result.title = test.testName;
result.passed = true;
} catch (err) {
result.time = Date.now() - result.time;
result.passed = false;
console.error(err.name);
console.error(err.message);
}
test_results.push(result)
post({
type: 'result',
title: test.testName,
result: result
});
result = {
'title': '',
'passed': false,
'stdOut': '',
'stdErr': '',
'time': 0
'stdErr': ''
};
}
callback(test_results);
callback({
type: 'suite end',
result: result
});
};
module.exports.run_tests = run_tests;
79 changes: 50 additions & 29 deletions Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ var result = {
'title': '',
'passed': false,
'stdOut': '',
'stdErr': '',
'time': 0
'stdErr': ''
};

function append_stdout(string, encoding, fd) {
result.stdOut += string;
}

function append_stderr(string, encoding, fd) {
result.stdErr += string;
}
Expand Down Expand Up @@ -52,42 +52,63 @@ function find_tests(testFileList, discoverResultFile, projectFolder) {
module.exports.find_tests = find_tests;

function run_tests(testInfo, callback) {
var testResults = [];
var testCases = loadTestCases(testInfo[0].testFile);
process.stdout.write = append_stdout;
process.stderr.write = append_stderr;
if (testCases === null) {
return;
}

var tape = findTape(testInfo[0].projectFolder);
if (tape === null) {
return;
}

for (var test in testInfo) {
result.title = testInfo[test].testName;
var harness = tape.getHarness();

testInfo.forEach(function (info) {
runTest(info, harness, function (result) {
callback(result);
});
});

tape.onFinish(function () {
// executes when all tests are done running
});

function runTest(testInfo, harness, done) {
var stream = harness.createStream({ objectMode: true });
var title = testInfo.testName;

stream.on(('data'), function (result) {
if (result.type === 'test') {
done({
type: 'test start',
title: title
});
}
});

try {
result.time = Date.now();
var harness = tape.getHarness();
harness(testInfo[test].testName);
result.passed = true;
var htest = tape.test(title, {}, function (result) {
done({
type: 'result',
title: title,
result: {
'title': title,
'passed': result._ok,
'stdOut': '',
'stdErr': ''
}
});
});
} catch (e) {
result.passed = false;
logError('Error running test:', testInfo[test].testName, 'in', testInfo[test].testFile, e);
console.error('NTVS_ERROR:', e);
done({
type: 'result',
title: title,
result: {
'title': title,
'passed': false,
'stdOut': '',
'stdErr': e.message
}
});
}
result.time = Date.now() - result.time;
testResults.push(result);
result = {
'title': '',
'passed': false,
'stdOut': '',
'stdErr': '',
'time': 0
};
}

callback(testResults);
}
module.exports.run_tests = run_tests;

Expand Down
94 changes: 74 additions & 20 deletions Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ var result = {
'title': '',
'passed': false,
'stdOut': '',
'stdErr': '',
'time': 0
'stdErr': ''
};
// Choose 'tap' rather than 'min' or 'xunit'. The reason is that
// 'min' produces undisplayable text to stdout and stderr under piped/redirect,
Expand All @@ -19,8 +18,12 @@ function append_stdout(string, encoding, fd) {
function append_stderr(string, encoding, fd) {
result.stdErr += string;
}
process.stdout.write = append_stdout;
process.stderr.write = append_stderr;
function hook_outputs() {
process.stdout.write = append_stdout;
process.stderr.write = append_stderr;
}

hook_outputs();

var find_tests = function (testFileList, discoverResultFile, projectFolder) {
var Mocha = detectMocha(projectFolder);
Expand Down Expand Up @@ -71,6 +74,11 @@ var find_tests = function (testFileList, discoverResultFile, projectFolder) {
module.exports.find_tests = find_tests;

var run_tests = function (testCases, callback) {
function post(event) {
callback(event);
hook_outputs();
}

function escapeRegExp(string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
}
Expand All @@ -86,49 +94,95 @@ var run_tests = function (testCases, callback) {
var testGrepString = '^(' + testCases.map(function (testCase) {
return testCase.testName
}).join('|') + ')$';

if (testGrepString) {
mocha.grep(new RegExp(testGrepString));
}

mocha.addFile(testCases[0].testFile);

// run tests
var runner = mocha.run(function (code) { });
var runner = mocha.run(function (code) {
process.exit(code);
});

runner.on('suite', function (suite) {
post({
type: 'suite start',
result: result
});
});

runner.on('suite end', function (suite) {
post({
type: 'suite end',
result: result
});
});

runner.on('hook', function (hook) {
post({
type: 'hook start',
title: hook.title,
result: result
});
});

runner.on('hook end', function (hook) {
post({
type: 'hook end',
title: hook.title,
result: result
});
});

runner.on('start', function () {
post({
type: 'start',
result: result
});
});

runner.on('test', function (test) {
result.title = test.fullTitle();
result.time = Date.now();
process.stdout.write = append_stdout;
process.stderr.write = append_stderr;
post({
type: 'test start',
title: result.title
});
});

runner.on('end', function () {
callback(testResults);
post({
type: 'end',
result: result
});
});

runner.on('pass', function (test) {
result.passed = true;
result.time = Date.now() - result.time;
testResults.push(result);
post({
type: 'result',
title: result.title,
result: result
});
result = {
'title': '',
'passed': false,
'stdOut': '',
'stdErr': '',
'time': ''
'stdErr': ''
}
});

runner.on('fail', function (test, err) {
result.passed = false;
result.time = Date.now() - result.time;
testResults.push(result);
post({
type: 'result',
title: result.title,
result: result
});
result = {
'title': '',
'passed': false,
'stdOut': '',
'stdErr': '',
'time': ''
'stdErr': ''
}
});
};
Expand Down
6 changes: 2 additions & 4 deletions Nodejs/Product/Nodejs/TestFrameworks/run_tests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,16 @@ rl.on('line', (line) => {
process.exit(1);
}

function returnResult(result) {
function postResult(result) {
// unhook stdout and stderr
process.stdout.write = old_stdout;
process.stderr.write = old_stderr;
if (result) {
console.log(JSON.stringify(result));
}
// end process, tests are done running.
process.exit(0);
}
// run the test
framework.run_tests(testCases, returnResult);
framework.run_tests(testCases, postResult);

// close readline interface
rl.close();
Expand Down
Loading