Merge pull request #1260 from mozilla/issue1245

by default, run all tests during npm test, and emit clear errors when de...

README.md

@@ -42,11 +42,15 @@ You can stop the servers with a Cntl-C in the terminal.
 
 ## Testing
 
-Unit tests can be run by invoking `npm test` at the top level, and you
-should run them often.  Like before committing code.  To fully test
-the code you should install mysql and have a well permissions `test`
-user (can create and drop databases).  If you don't have mysql installed,
-code testing is still possible (it just uses a little JSON database).
+Unit tests can be run by invoking `npm test` at the top level.  At present,
+there are three classes of unit tests to be run:
+
+  * Backend unit tests against a custom, zero-dependency JSON database
+  * Backend unit tests against MySQL, what we use in production
+  * Frontend unit tests run headlessly against PhantomJS
+
+You can control which tests are run using the `WHAT_TESTS` env var, see
+`scripts/test` for details.
 
 ## Development model
 

scripts/test

@@ -2,8 +2,9 @@
 
 // a script to RUN TESTS.  You can specify WHAT TESTS to run by
 // populating an environment variable 'WHAT_TESTS'.  Values include:
-//   * 'front' - frontend unit tests
-//   * 'back' - backend unit tests
+//   * 'front' - frontend unit tests run headlessly (requires phantomjs to be installed)
+//   * 'back' - backend unit tests with a zero-dependency json db
+//   * 'back_mysql - backend unit tests against mysql (requires mysql installed)
 //   * 'all' - of it
 
 const
@@ -12,28 +13,53 @@ path = require('path');
 
 // WHAT TESTS are we running?
 var whatTests = [];
-if (process.env['WHAT_TESTS']) {
-  whatTests.push(process.env['WHAT_TESTS']);
-  if (whatTests[0] == 'all') whatTests = [ 'front', 'back' ];
+if (!process.env['WHAT_TESTS']) {
+  whatTests = [ 'all' ];
 } else {
-  whatTests = [ 'back' ];
+  whatTests.push(process.env['WHAT_TESTS']);
 }
 
+if (whatTests[0] == 'all') whatTests = [ 'back_mysql', 'back', 'front' ];
+
 var ec = 0;
 function run() {
   if (!whatTests.length) process.exit(ec);
 
-  var script = {
-    front: 'test_frontend',
-    back: 'test_backend'
-  }[whatTests.shift()];
+  var testName = whatTests.shift();
+
+  const availConf = {
+    front: {
+      what: "Front end unit tests under PhantomJS",
+      node_env: 'test_json',
+      script: 'test_frontend'
+    },
+    back: {
+      what: "API level unit tests using a JSON database",
+      node_env: 'test_json',
+      script: 'test_backend'
+    },
+    back_mysql: {
+      what: "API level unit tests using MySQL",
+      node_env: 'test_mysql',
+      script: 'test_backend'
+    }
+  };
+  var conf = availConf[testName];
+
+  if (!conf) {
+    console.log('I don\'t know how to run this test: "' + testName + '"');
+    console.log('valid choices include:', Object.keys(availConf).join(', '));
+    ec = 1;
+    return run();
+  }
 
-  console.log(script);
-  var kid = spawn(path.join(__dirname, script));
+  console.log(">>> Now Running:", conf.what);
+  process.env['NODE_ENV'] = conf.node_env;
+  var kid = spawn(path.join(__dirname, conf.script));
   kid.stdout.on('data', function(d) { process.stdout.write(d); });
   kid.stderr.on('data', function(d) { process.stderr.write(d); });
   kid.on('exit', function(code) {
-    if (code) process.exit(code);
+    if (code) ec = code;
     run();
   });
 }

scripts/test_backend

@@ -27,5 +27,8 @@ if [ $? = 0 ] ; then
         fi
     done
 else
-    echo "CANNOT TEST '$env' ENVIRONMENT: can't connect to the database"
+    echo
+    echo "Can't run tests: can't connect to the database"
+    echo
+    exit 1
 fi

scripts/test_db_connectivity.js

@@ -22,8 +22,8 @@ var dbCfg = configuration.get('database');
 delete dbCfg.create_schema;
 
 db.open(dbCfg, function (err, r) {
-  if (err && err.message === "Unknown database 'browserid'") r = undefined;
-  function end() { process.exit(r === undefined ? 0 : 1); }
-  if (r === undefined) db.close(end);
+  if (err && err.message === "Unknown database 'browserid'") err = undefined;
+  function end() { process.exit(err ? 1 : 0); }
+  if (!err) db.close(end);
   else end();
 });

scripts/test_frontend

@@ -18,6 +18,18 @@ var suite = vows.describe('frontend-tests');
 // disable vows (often flakey?) async error behavior
 suite.options.error = false;
 
+suite.addBatch({
+  "PhantomJS binary": {
+    topic: function() {
+      var kid = spawn('phantomjs', [ '--version' ]);
+      kid.on('exit', this.callback);
+    },
+    "is in path and executable": function(code) {
+      assert.strictEqual(code, 0);
+    }
+  }
+});
+
 start_stop.addStartupBatches(suite);
 
 suite.addBatch({