Test adapter improvements

Common/Product/SharedProject/ProcessOutput.cs

@@ -58,6 +58,15 @@ public virtual void Show() {
         /// </summary>
         public virtual void ShowAndActivate() {
         }
+
+        /// <summary>
+        /// Called to determine if stdin should be closed for a redirected process.
+        /// The default is true.
+        /// </summary>
+        public virtual bool CloseStandardInput()
+        {
+            return true;
+        }
     }
 
     sealed class TeeRedirector : Redirector, IDisposable {
@@ -304,7 +313,7 @@ public static ProcessOutput RunElevated(
             return result;
         }
 
-        private static string GetArguments(IEnumerable<string> arguments, bool quoteArgs) {
+        public static string GetArguments(IEnumerable<string> arguments, bool quoteArgs) {
             if (quoteArgs) {
                 return string.Join(" ", arguments.Where(a => a != null).Select(QuoteSingleArgument));
             } else {
@@ -335,7 +344,7 @@ internal static IEnumerable<string> SplitLines(string source) {
             }
         }
 
-        internal static string QuoteSingleArgument(string arg) {
+        public static string QuoteSingleArgument(string arg) {
             if (string.IsNullOrEmpty(arg)) {
                 return "\"\"";
             }
@@ -423,10 +432,16 @@ private ProcessOutput(Process process, Redirector redirector) {
 
                 if (_process.StartInfo.RedirectStandardInput) {
                     // Close standard input so that we don't get stuck trying to read input from the user.
-                    try {
-                        _process.StandardInput.Close();
-                    } catch (InvalidOperationException) {
-                        // StandardInput not available
+                    if (_redirector == null || (_redirector != null && _redirector.CloseStandardInput()))
+                    {
+                        try
+                        {
+                            _process.StandardInput.Close();
+                        }
+                        catch (InvalidOperationException)
+                        {
+                            // StandardInput not available
+                        }
                     }
                 }
             }
@@ -557,6 +572,20 @@ public Redirector Redirector {
             get { return _redirector; }
         }
 
+        /// <summary>
+        /// Writes a line to stdin. A redirector must have been provided that indicates not
+        /// to close the StandardInput stream.
+        /// </summary>
+        /// <param name="line"></param>
+        public void WriteInputLine(string line)
+        {
+            if (IsStarted && _redirector != null && !_redirector.CloseStandardInput())
+            {
+                _process.StandardInput.WriteLine(line);
+                _process.StandardInput.Flush();
+            }
+        }
+
         private void FlushAndCloseOutput() {
             if (_process == null) {
                 return;

Common/Product/TestAdapter/VisualStudioApp.cs

@@ -15,7 +15,9 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Linq;
+using System.Reflection;
 using System.Runtime.InteropServices;
 using EnvDTE;
 using Microsoft.VisualStudio.OLE.Interop;
@@ -75,7 +77,35 @@ public DTE GetDTE() {
             return dte;
         }
 
+#if DEV15
+        private static bool DTELoaded = false;
+#endif
+
         private static DTE GetDTE(int processId) {
+#if DEV15
+            // VS 2017 doesn't install some assemblies to the GAC that are needed to work with the
+            // debugger, and as the tests don't execute in the devenv.exe process, those assemblies
+            // fail to load - so load them manually from PublicAssemblies.
+
+            // Use the executable name, as this is only needed for the out of proc test execution
+            // that may interact with the debugger (vstest.executionengine.x86.exe).
+            if (!DTELoaded)
+            {
+                string currentProc = Process.GetCurrentProcess().MainModule.FileName;
+                if (StringComparer.OrdinalIgnoreCase.Equals(
+                        Path.GetFileName(currentProc), "vstest.executionengine.x86.exe"))
+                {
+                    string baseDir = Path.GetDirectoryName(currentProc);
+                    string publicAssemblies = Path.Combine(baseDir, "..\\..\\..\\PublicAssemblies");
+
+                    Assembly.LoadFrom(Path.Combine(publicAssemblies, "Microsoft.VisualStudio.OLE.Interop.dll"));
+                    Assembly.LoadFrom(Path.Combine(publicAssemblies, "envdte90.dll"));
+                    Assembly.LoadFrom(Path.Combine(publicAssemblies, "envdte80.dll"));
+                    Assembly.LoadFrom(Path.Combine(publicAssemblies, "envdte.dll"));
+                }
+                DTELoaded = true;
+            }
+#endif
             MessageFilter.Register();
 
             var prefix = Process.GetProcessById(processId).ProcessName;

Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js

@@ -1,6 +1,26 @@
 var fs = require('fs');
 var path = require('path');
 var vm = require('vm');
+var result = {
+    'title': '',
+    'passed': false,
+    'stdOut': '',
+    'stdErr': ''
+};
+
+function append_stdout(string, encoding, fd) {
+    result.stdOut += string;
+}
+function append_stderr(string, encoding, fd) {
+    result.stdErr += string;
+}
+function hook_outputs() {
+    process.stdout.write = append_stdout;
+    process.stderr.write = append_stderr;
+}
+
+
+hook_outputs();
 
 var find_tests = function (testFileList, discoverResultFile) {
     var debug;
@@ -50,8 +70,43 @@ var find_tests = function (testFileList, discoverResultFile) {
 };
 module.exports.find_tests = find_tests;
 
-var run_tests = function (testName, testFile) {
-    var testCase = require(testFile);
-    testCase[testName]();
+var run_tests = function (testCases, callback) {
+    function post(event) {
+        callback(event);
+        hook_outputs();
+    }
+
+    for (var test of testCases) {
+        post({
+            type: 'test start',
+            title: test.testName
+        });
+        try {
+            var testCase = require(test.testFile);
+            result.title = test.testName;
+            testCase[test.testName]();
+            result.passed = true;
+        } catch (err) {
+            result.passed = false;
+            console.error(err.name);
+            console.error(err.message);
+        }
+        post({
+            type: 'result',
+            title: test.testName,
+            result: result
+        });
+        result = {
+            'title': '',
+            'passed': false,
+            'stdOut': '',
+            'stdErr': ''
+        };
+    }
+    callback({
+        type: 'suite end',
+        result: result
+    });
+	process.exit();
 };
 module.exports.run_tests = run_tests;

Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js

@@ -2,6 +2,20 @@
 var EOL = require('os').EOL;
 var fs = require('fs');
 var path = require('path');
+var result = {
+    'title': '',
+    'passed': false,
+    'stdOut': '',
+    'stdErr': ''
+};
+
+function append_stdout(string, encoding, fd) {
+    result.stdOut += string;
+}
+
+function append_stderr(string, encoding, fd) {
+    result.stdErr += string;
+}
 
 function find_tests(testFileList, discoverResultFile, projectFolder) {
     var test = findTape(projectFolder);
@@ -10,7 +24,7 @@ function find_tests(testFileList, discoverResultFile, projectFolder) {
     }
 
     var harness = test.getHarness({ exit: false });
-    var tests = harness["_tests"];
+    var tests = harness['_tests'];
 
     var count = 0;
     var testList = [];
@@ -37,24 +51,78 @@ function find_tests(testFileList, discoverResultFile, projectFolder) {
 };
 module.exports.find_tests = find_tests;
 
-function run_tests(testName, testFile, workingFolder, projectFolder) {
-    var testCases = loadTestCases(testFile);
-    if (testCases === null) {
+function run_tests(testInfo, callback) {
+    var tape = findTape(testInfo[0].projectFolder);
+    if (tape === null) {
         return;
     }
 
-    var test = findTape(projectFolder);
-    if (test === null) {
-        return;
-    }
+    var harness = tape.getHarness({objectMode: true});
+    var capture = false; // Only capture between 'test' and 'end' events to avoid skipped test events.
+    harness.createStream({ objectMode: true }).on('data', function (evt){
+        switch (evt.type) {
+            case 'test':
+                capture = true;
+                // Test is starting. Reset the result object. Send a "test start" event.
+                result = {
+                    'title': evt.name,
+                    'passed': true,
+                    'stdOut': '',
+                    'stdErr': ''
+                };
+                callback({
+                    'type': 'test start',
+                    'title': result.title,
+                    'result': result
+                });
+                break;
+            case 'assert':
+                if (!capture) break;
+                // Correlate the success/failure asserts for this test. There may be multiple per test
+                var msg = "Operator: " + evt.operator + ". Expected: " + evt.expected + ". Actual: " + evt.actual + "\n";
+                if (evt.ok) {
+                    result.stdOut += msg;
+                } else {
+                    result.stdErr += msg + (evt.error.stack || evt.error.message) + "\n";
+                    result.passed = false;
+                }
+                break;
+            case 'end':
+                if (!capture) break;
+                // Test is done. Send a "result" event.
+                callback({
+                    'type': 'result',
+                    'title': result.title,
+                    'result': result
+                });
+                capture = false;
+                break;
+            default:
+                break;
+        }
+    });
 
-    try {
-        var harness = test.getHarness();
-        harness.only(testName);
-    } catch (e) {
-        logError("Error running test:", testName, "in", testFile, e);
-        return;
-    }
+    loadTestCases(testInfo[0].testFile);
+
+    // Skip those not selected to run. The rest will start running on the next tick.
+    harness['_tests'].forEach(function(test){
+        if( !testInfo.some( function(ti){ return ti.testName == test.name; }) ) {
+            test._skip = true;
+        }
+    });
+
+    harness.onFinish(function () {
+        if (capture) {
+            // Something didn't finish. Finish it now.
+            result.passed = false;
+            callback({
+                'type': 'result',
+                'title': result.title,
+                'result': result
+            });
+        }
+        process.exit(0);
+    });
 }
 module.exports.run_tests = run_tests;