diff --git a/automation-tests/lib/convert_results.js b/automation-tests/lib/convert_results.js
new file mode 100755
index 0000000000000000000000000000000000000000..fb7a78929a4debba68afe4d1f179435891df190c
--- /dev/null
+++ b/automation-tests/lib/convert_results.js
@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+/*
+ * Converts html reports into nice, machine readable JSON
+ * Run: $ ./convert_result.js result/index.html
+ */
+
+const fs = require('fs'),
+ path = require('path'),
+ jsonselect = require('JSONSelect'),
+ htmlparser = require('htmlparser');
+
+
+function main (args) {
+ var file = fs.readFile(path.resolve(args[2]), "utf8", function (err, html) {
+ if (err) throw err;
+ parseReport(html);
+ });
+}
+
+function parseReport (html) {
+ var report = {};
+ var handler = new htmlparser.DefaultHandler(function(err, dom) {
+ if (err) {
+ console.error("Error: " + err);
+ } else {
+ var results = jsonselect.match(':has(:root > .attribs > .id:val("results")) .children :has(:root > .name:val("tr"))', dom);
+
+ // remove header row
+ results.shift();
+
+ results.forEach(function (node, i, array) {
+ var url;
+ var result = node.children[1].attribs.class;
+
+ // skip traceback rows
+ if (!result) return;
+
+ try {
+ url = result === 'error' ?
+ findJobUrl(array[i+1].children[1].children[1].children) :
+ node.children[9].children[0].attribs.href;
+ } catch (e) {
+ url = '';
+ }
+
+ var name = node.children[5].children[0].data;
+
+ report[name] = {
+ success: result === 'passed',
+ class: node.children[3].children[0].data,
+ duration: node.children[7].children[0].data,
+ url: url
+ };
+ });
+ }
+ });
+
+ var parser = new htmlparser.Parser(handler);
+ parser.parseComplete(html);
+ return report;
+}
+
+// extract saucelab url from error report
+function findJobUrl (children) {
+ var result;
+ children.forEach(function (node) {
+ var match = node.raw.match(/https:\/\/saucelabs.com\/jobs\/[a-f0-9]+/);
+ if (match) result = match[0];
+ });
+ return result;
+}
+
+exports.parseReport = parseReport;
+
+if (process.argv[1] === __filename) main(process.argv);
diff --git a/automation-tests/run_saucelabs b/automation-tests/run_saucelabs
new file mode 100755
index 0000000000000000000000000000000000000000..68e34ebd9976613f6f2d9af643def86920845bab
--- /dev/null
+++ b/automation-tests/run_saucelabs
@@ -0,0 +1,364 @@
+#!/usr/bin/env node
+
+var child_process = require('child_process'),
+ path = require('path'),
+ _ = require('underscore'),
+ fs = require('fs'),
+ glob = require('minimatch'),
+ temp = require('temp'),
+ which = require('which'),
+ parseReport = require('./lib/convert_results.js').parseReport,
+ events = require('events'),
+ util = require('util'),
+ mkdirp = require('mkdirp');
+
+function runCmd(cmd, opts, cb) {
+ if (!cb) {
+ cb = opts;
+ opts = { cwd: path.dirname(__dirname) };
+ }
+ var cp = child_process.exec(cmd, opts, function(err, stdout, stderr) {
+ cb(err, stdout, stderr);
+ });
+}
+
+function TestRunner() {
+ events.EventEmitter.call(this);
+}
+
+util.inherits(TestRunner, events.EventEmitter);
+
+// path to automation_tests
+const testPath = path.join(path.dirname(__dirname), "automation-tests");
+
+// ephemeral files to hold various credentials desired by py.test gunk
+const sauceYAMLPath = temp.path({suffix: '.yaml'}),
+ credentialsYAMLPath = temp.path({suffix: '.yaml'});
+
+// python arguments common to all tests
+var globalPythonArgs = {
+ "-m": "py.test",
+ "--credentials": credentialsYAMLPath,
+ "--saucelabs": sauceYAMLPath,
+ "--webqatimeout": 90,
+ "--destructive": null,
+ "-q": null,
+ '--capabilities': JSON.stringify({ "avoid-proxy":"true"})
+};
+
+// python arguments specific to different test classes
+var testSpecificPythonArgs = {
+ "123done": {
+ "--baseurl": "http://dev.123done.org"
+ },
+ "browserid": {
+ "--baseurl": "http://dev.123done.org"
+ },
+ "myfavoritebeer": {
+ "--baseurl": "http://dev.myfavoritebeer.org"
+ }
+};
+
+var browserSpecificPythonArgs = {
+ "linux_firefox_13": {
+ '--platform': 'LINUX',
+ '--browsername': 'firefox',
+ '--browserver': '13'
+ },
+ "linux_opera_12": {
+ '--platform': 'LINUX',
+ '--browsername': 'opera',
+ '--browserver': '12'
+ },
+ "osx_firefox_14": {
+ '--platform': 'MAC',
+ '--browsername':'firefox',
+ '--browserver':'14'
+ },
+ "vista_chrome": {
+ '--platform':'VISTA',
+ '--browsername':'chrome'
+ },
+ "vista_firefox_13": {
+ '--platform':'VISTA',
+ '--browsername':'firefox',
+ '--browserver':'13'
+ },
+ "vista_ie_9": {
+ '--platform':'VISTA',
+ '--browsername':'internet explorer',
+ '--browserver':'9'
+ },
+ "xp_ie_8": {
+ '--platform':'XP',
+ '--browsername': 'internet explorer',
+ '--browserver':'8'
+ }
+};
+
+function escape(val) {
+ return '"'+val.replace(/(["'$`\\])/g,'\\$1')+'"';
+};
+
+// now write a yaml file with sauce creds
+function writeSauceYAML() {
+ var envVars = {
+ 'PERSONA_SAUCE_USER': 'username',
+ 'PERSONA_SAUCE_PASSWORD': 'password',
+ 'PERSONA_SAUCE_APIKEY': 'api-key'
+ };
+
+ var fileContents = "";
+ Object.keys(envVars).forEach(function(key) {
+ if (!process.env[key]) throw "missing sauce labs creds from environment";
+ fileContents += envVars[key] + ": " + process.env[key] + "\n"
+ });
+ fs.writeFileSync(sauceYAMLPath, fileContents);
+}
+
+// now write a yaml file with sauce creds
+function writeCredsYAML() {
+ var envVars = {
+ 'PERSONA_EMAIL': 'email',
+ 'PERSONA_PASSWORD': 'password'
+ };
+
+ var fileContents = "default:\n";
+ Object.keys(envVars).forEach(function(key) {
+ if (!process.env[key]) throw "missing exisiting users creds from environment: " + key;
+ fileContents += " " + envVars[key] + ": " + process.env[key] + "\n"
+ });
+ fs.writeFileSync(credentialsYAMLPath, fileContents);
+}
+
+// setup python testing environment iff required, return path to python
+TestRunner.prototype._setupPythonEnv = function(cb) {
+ var binPath = path.join(testPath, "bid_selenium", "bin");
+
+ function isSetup(lcb) {
+ var pathToPython = path.join(binPath, "python");
+ fs.stat(pathToPython, function(err, r) {
+ if (!err && r.isFile()) lcb(null, { pathToPython: pathToPython });
+ else lcb("not setup");
+ });
+ }
+ function findExecutable(names, cb) {
+ if (!names.length) return cb("not found");
+ var n = names.shift();
+ which(n, function(err, p) {
+ if (err) return findExecutable(names, cb);
+ cb(null, p);
+ });
+ }
+
+ isSetup(function(err, r) {
+ if (!err) return cb(null, r);
+
+ // time to set it up!
+ findExecutable([ 'virtualenv-2.7', 'virtualenv2', 'virtualenv' ], function(err, virtualenv) {
+ if (err) return cb("cannot find virtualenv");
+ runCmd(virtualenv + " " + path.join(testPath, "bid_selenium"), function(err, stdout) {
+ if (err) return cb("cannot run virtualenv: " + err);
+ runCmd(path.join(binPath, "pip") + " install -Ur requirements.txt", { cwd: testPath }, function(err, stdout) {
+ if (err) cb(err);
+ else isSetup(cb);
+ });
+ });
+ });
+ });
+}
+
+TestRunner.prototype.run = function(opts, cb) {
+ var self = this;
+ var processesRunning = 0;
+ var testReports = [];
+ var overallStartTime = new Date();
+
+ // once all tests are complete, crawl through the data and write
+ // summary information
+ function addSummaryInfo(results) {
+ // XXX: calculcate browser and tests summary information
+ return {
+ duration: ((new Date() - overallStartTime) / 1000.0),
+ reports: results
+ };
+ }
+
+ opts = opts || {};
+
+ writeSauceYAML();
+ writeCredsYAML();
+
+ this._setupPythonEnv(function(err, testEnvDetails) {
+ if (err) {
+ console.log("ERROR: couldn't setup python environment");
+ return cb(err);
+ }
+ Object.keys(browserSpecificPythonArgs).forEach(function(browser) {
+ if (opts.browser && !glob(browser, opts.browser.toString())) return;
+ var browserArgs = browserSpecificPythonArgs[browser];
+
+ Object.keys(testSpecificPythonArgs).forEach(function(test) {
+ if (opts['test-group'] && !glob(test, (opts['test-group']).toString())) return;
+
+ var htmlReportPath = temp.path({suffix: '.html'});
+ var testArgs = testSpecificPythonArgs[test];
+
+ // build up the command line arguments
+ var cmdargsObj = {};
+ _.extend(cmdargsObj, globalPythonArgs, testArgs, browserArgs,
+ { '--webqareport': htmlReportPath });
+ var cmdargs = "";
+ Object.keys(cmdargsObj).forEach(function(flag) {
+ var spc = " ";
+ if (flag.substr(2) === '--') spc = "=";
+ if (null === cmdargsObj[flag]) {
+ cmdargs += flag + " ";
+ } else {
+ cmdargs += flag + spc + escape(cmdargsObj[flag].toString()) + " ";
+ }
+ });
+ if (opts.single) {
+ cmdargs += " " + path.relative(testPath, opts.single);
+ } else {
+ cmdargs += " " + test;
+ }
+ self.emit('started', { browser: browser, test: test });
+ var startTime = new Date();
+ processesRunning++;
+ runCmd(testEnvDetails.pathToPython + " " + cmdargs, { cwd: testPath }, function(err, stdout, stderr) {
+ var report = {
+ browser: browser,
+ test: test,
+ duration: ((new Date() - startTime) / 1000.0),
+ stdout: stdout,
+ stderr: stderr,
+ err: err,
+ passed: !err
+ };
+
+ try {
+ report.htmlReport = fs.readFileSync(htmlReportPath);
+ } catch(e) { }
+
+ // extract key information from the html report and attach it to the report object
+ report.results = parseReport(report.htmlReport);
+
+ self.emit('finished', report);
+
+ testReports.push(report);
+
+ // remove artifacts
+ fs.unlink(htmlReportPath);
+
+ if (--processesRunning === 0) {
+ fs.unlink(sauceYAMLPath);
+ fs.unlink(credentialsYAMLPath);
+ if (cb) cb(null, addSummaryInfo(testReports));
+ }
+ });
+ });
+ });
+ });
+};
+
+module.exports = TestRunner;
+
+// if we're invoked from the command line, do command liney things
+if (process.argv[1] === __filename) {
+ process.on('uncaughtException', function(err) {
+ console.log("OH NOES", err);
+ process.exit(1);
+ });
+
+ var argv = require('optimist')
+ .usage('Run selenium tests via sauce labs.\nUsage: $0')
+ .alias('help', 'h')
+ .describe('help', 'display this usage message')
+ .alias('list-browsers', 'lb')
+ .describe('lb', 'list available browsers to test on')
+ .alias('browser', 'b')
+ .describe('browser', 'specify which browser to run tests on (globs supported)')
+ .alias('list-test-groups', 'lt')
+ .describe('list-test-groups', 'list available groups of tests that can be run')
+ .alias('test-group', 't')
+ .describe('test-group', 'specify which test groups to run (globs supported)')
+ .describe('single', 'run a single test within the specified test group')
+ .alias('single', 's')
+ .default('browser', "linux_firefox_13") // default to a specific browser + test group
+ .default("test-group", "123done"); // lest folks accidentally launch large #s of || jobs
+
+ var args = argv.argv;
+
+ if (args.h) {
+ argv.showHelp();
+ process.exit(0);
+ }
+
+ if (args.lb) {
+ console.log("available browsers:");
+ console.log(" * " + Object.keys(browserSpecificPythonArgs).join("\n * "));
+ process.exit(0);
+ }
+
+ if (args.lt) {
+ console.log("available tests:");
+ console.log(" * " + Object.keys(testSpecificPythonArgs).join("\n * "));
+ process.exit(0);
+ }
+
+ // nice file names
+ var startTime = new Date();
+ function testFileName(browser, test) {
+ function pad(n){return n<10 ? '0'+n : n}
+ var d = startTime;
+ var name = "" +
+ d.getFullYear() +
+ "." + pad(d.getMonth()+1)+'.'
+ + pad(d.getDate())+'-'
+ + pad(d.getHours())+':'
+ + pad(d.getMinutes())+':'
+ + pad(d.getSeconds());
+ name += "-" + browser + "-" + test;
+ return name;
+ }
+
+ var tester = new TestRunner();
+ tester.on('started', function(e) {
+ console.log("STARTED:", e.browser + "/" + e.test);
+ });
+
+ tester.on('finished', function(report) {
+ // save off the report files
+ var basename = testFileName(report.browser, report.test);
+ var resultsPath = path.join(testPath, "results", basename);
+ mkdirp.sync(path.join(testPath, "results"));
+ fs.writeFileSync(resultsPath + ".html", report.htmlReport);
+ if (report.stderr.length) fs.writeFileSync(resultsPath + ".stderr.txt", report.stderr);
+ if (report.stdout.length) fs.writeFileSync(resultsPath + ".stdout.txt", report.stdout);
+
+ var what = report.passed ? "PASSED" : "FAILED";
+ console.log(what + ": " + report.browser + "/" + report.test +
+ " - " + report.duration.toFixed(2) + "s (" +
+ path.relative(process.cwd, resultsPath + ".html") + ")");
+
+ if (!report.passed) {
+ Object.keys(report.results).forEach(function (resultName) {
+ var result = report.results[resultName];
+ if (result.success) return;
+ console.log(" " + result.class + ": " + result.url);
+ });
+ if (report.stderr.length) {
+ console.log(" ERRORS:\n > " + report.stderr.split("\n").join("\n > "));
+ }
+ }
+ });
+ tester.run(args, function(report) {
+ // We would like to summarize:
+ // 1. total test duration
+ // 2. number of browsers failing
+ // 3. number of tests failing
+ // XXX: after ALL tests complete output a short summary
+// console.log(JSON.stringify(report, null, " "));
+ });
+}
diff --git a/package.json b/package.json
index 97519eb6f52cd5f80825df93cbfa2a41134c5be2..a58c81dcc0e612b5a1e2d25e5d3dbdeb66e66908 100644
--- a/package.json
+++ b/package.json
@@ -40,7 +40,10 @@
"vows": "0.5.13",
"awsbox": "0.2.15",
"irc": "0.3.3",
- "jshint": "0.7.1"
+ "jshint": "0.7.1",
+ "minimatch": "0.2.6",
+ "which": "1.0.5",
+ "htmlparser": "1.7.6"
},
"scripts": {
"postinstall": "node ./scripts/generate_ephemeral_keys.js",