A simpler way to run tests?

[DinisCruz]

OK, when I had the #2741 issue, I was trying to create a simple 'test run' environment that gave me the 'mocha based TDD environment' that I like to code and that makes me very productive (as a programmer).

Basically what I want is to be able to write node-webkit tests in an IDE and have them executed in real-time using mocha (with the results shown in one of the build-in mocha reporters).

The challenge are:

• there are two host processes at play (the one starting node-webkit and the node-webkit+mocha)
• node-webkit needs to start a browser (to have access to the webkit part)
• the mocha tests must run inside the browser
• only with a full node-webkit + mocha environment are we able to write tests that access node+webkit APIs (for example 'nw-gui' and 'fs')

Here is the type of test that I wanted to write:

assert = require('assert')

describe('Node-WebKit Mocha tests', function()
{
  nw         = null
  get_Window = null

  it('Get nw.gui', function()
    {
       assert.equal(nw, null)
       nw = require('nw.gui')
       assert.notEqual(nw, null)
       assert.ok(nw.App)
       assert.ok(nw.Shell)
       assert.ok(nw.Window)
    });

  it('Get window reference', function()
    {
       get_Window = nw.Window.get()
       assert.notEqual(get_Window,null)
       assert.ok(get_Window.routing_id)
       assert.ok(get_Window.cookies)
    })
  it('Check window title', function()
    {
       title_From_Dom        = document.title
       title_From_Get_Window = get_Window.title
       assert.equal(title_From_Dom,'Node-WebKit for mocha')
       assert.equal(title_From_Dom, title_From_Dom)
     })
  //remove the x to see this fail
  xit('this will fail', function()
    {
      assert.equal(1,2)
    });
});

After a while, I was able to figure out a way to do, which (maybe) might become a simpler way to run/develope some of the node-webkit tests.

Here is the approach/techniques I used (note that the code is still quite rough around the edges)

1. use a coffee-script to start nodewebkit executable and set up watcher on a couple folders (in the beginning I used mocha and its watcher to automate the re-execution of nodewebkit process)
2. very simple index.html page, which is open up with show: false, and loads up a background JS file
3. JS file (loaded from index.html) that sets-up and hosts mocha
4. write normal mocha tests inside a ./tests folder (with auto-execution of tests on file changes)
5. see normal (colour coded) mocha results in the console out

Here are the files that implement this:

test-runner.coffee

require 'fluentnode'
fs = require('fs');

#method to add to fluentnode-child_process
String::start_Process = (args...)->
  args ?= []
  childProcess = require('child_process').spawn(@.str(),args)
  childProcess.stdout.on 'data', (data)->console.log(filterLog(data.str().trim()))
  childProcess.stderr.on 'data', (data)->console.log(filterLog(data.str().trim()))
  return childProcess


filterLog = (data)->
  if data.contains('breakpad_mac.mm(238)] Breakpad initializaiton failed')
    return ''
  return data

  ###
  # this can also be used to clean up the log messages routed from nodewebkit
  text = data.after('] "').before('", source')

  r = /\\u([\d\w]{4})/gi;
  x = text.replace r, (match, grp) -> String.fromCharCode(parseInt(grp, 16))
  unescape(x);
  if x == ''
    return data
  else
    x
  ###


console.log 'Monitoring for file changes in ./ and test'

monitorAndRun = (target)=>

  fs.watch target , (type)=>
    console.log("detected #{type} in #{target}, running nodewebkit")
    'nodewebkit'.start_Process()

monitorAndRun('./')
monitorAndRun('./tests')


'nodewebkit'.start_Process()



return

# below is another way to do it
# execute code with:
#    node ./node_modules/mocha/bin/mocha --compilers coffee:coffee-script/register test-runner.coffee -w


nodeWebKit = null


describe 'main',->
  before ->
    #'starting nodeWebKit'.log();
    nodeWebKit = 'nodewebkit'.start_Process()

  after ->
    #"stopping all 'node-webkit' processes".log()

    # this is a bit extreme way of doing this since we also kill any other node-webkit running in the current box
    # but since chome-webkit starts a couple processes, the .kill process alternatives where not working
    #'pkill'.start_Process('node-webkit')

    #nodeWebKit.kill('SIGINT');
    #process.kill(nodeWebKit.pid)

  it 'run nodewebkit',()->
      #'nodeWebKit should had started'.log();
#      done()

package.json

{
  "name": "nw-tests",
  "version": "0.1.0",
  "main": "index.html",
  "window": {
    "position": "center",
    "show": false
  },
  "scripts": {
    "test": "node ./node_modules/mocha/bin/mocha --compilers coffee:coffee-script/register test-runner.coffee -w"
  },
  "dependencies": {
    "coffee-script": "^1.8.0",
    "fluentnode": "^0.2.1"
  }
}

index.html

<html>
    <head>
      <title>Node-WebKit for mocha</title>      
        <script src='setup.js'></script>
    </head>
    <body>
        <h2>running mocha tests</h2>
    </body>
</html>

setup.js

util = require('util')
fs   = require('fs')
path = require('path')

//make messages more friendly on the parent process console
hookConsole = function()
  {
    console_log_writer = function(message,args) { process.stdout.write(util.format.apply(this,arguments)) }
    console.error = console_log_writer
    console.log   = console_log_writer
  }

//get files from tests folder
get_Test_Files = function()
  {
    return fs.readdirSync('tests').filter(function (file)
      {
          return file.substr(-3) === '.js';
      });
  }

//queue targetFiles tests
queue_Tests = function(targetFiles)
  {
    targetFiles.forEach(function (file)
    {
      jsFile = path.join('tests', file);
      jsCode= fs.readFileSync(jsFile, 'utf8');
      new Function(jsCode)();
    })
  }

setup_and_Run_Mocha = function(test_Files)
{
  var Mocha = require('mocha')
  var mocha = new Mocha;
  mocha.suite.emit('pre-require', window, null, mocha);

  queue_Tests(test_Files)
  return mocha.run(function (failures)
    {
      require('nw.gui').Window.get().close()
    });
}

hookConsole()
var test_Files = get_Test_Files()
var runner     = setup_and_Run_Mocha(test_Files)


//events that can be hooked
//mocha.suite.beforeEach(function() {} )
//mocha.suite.afterEach(function() {} )
//mocha.suite.afterAll( function() {} )

//runner.on('end', function() { console.log(data)})
//runner.on('suite', function(data) { console.log(data)})
//runner.on('fail', function(data) { console.log(data)})
//runner.on('test', function(data) { console.log(data)})

for reference (since this is the same test that was included in the beginning of this issue), here are the tests executed

./tests/mocha-test.js

assert = require('assert')

describe('Node-WebKit Mocha tests', function()
{
  nw         = null
  get_Window = null

  it('Get nw.gui', function()
    {
       assert.equal(nw, null)
       nw = require('nw.gui')
       assert.notEqual(nw, null)
       assert.ok(nw.App)
       assert.ok(nw.Shell)
       assert.ok(nw.Window)
    });

  it('Get window reference', function()
    {
       get_Window = nw.Window.get()
       assert.notEqual(get_Window,null)
       assert.ok(get_Window.routing_id)
       assert.ok(get_Window.cookies)
    })
  it('Check window title', function()
    {
       title_From_Dom        = document.title
       title_From_Get_Window = get_Window.title
       assert.equal(title_From_Dom,'Node-WebKit for mocha')
       assert.equal(title_From_Dom, title_From_Dom)
     })
  //remove the x to see this fail
  xit('this will fail', function()
    {
      assert.equal(1,2)
    });
});

Screenshots

1. IDE
   

2. Test execution (no failed tests)

3. Test execution (with failed test)
   

What is really cool about this setup (as you can see by the screenshots) is how fast the tests execute (i.e. sub 50ms), specially since there is not UI impact or flickering (due to the fact that the node-webkit window is opened in a transparent state)

Want to back this issue? Post a bounty on it! We accept bounties via Bountysource.

[rogerwang]

CC @yejingfu

[yejingfu]

Hello @DinisCruz To make sure we are on the same page, I want to explain some basic goals about the automation tests in node-webkit:

1. Each unit test is standalone nw application, since the test is targeting node-webkit. We don't want to treat individual js files as test units. That means we don't need to gather all individual js files and run them in sequence. Instean we treat each sub folder as a unit test (or standalone nw application).
2. Each unit test (e.g. nw application) should not affect the entire test framework. So far we have over 50 unit tests. It's very bad if one of them would block the entire test process when it's hanging.
3. Currently we write each unit test by mocha library. However mocha is not mandatory. It's up to each unit test to choose what technology be adopted.

Basically your implemenation is good reference. We still need to figure out a stable solution to run the big batch of unit test and meanwhile match the goal.
Thank you for your suggestions.
yejingfu