diff --git a/bin/router b/bin/router
index 56db15569691be00b3498ca77ddffe1171575f22..dc409f51267387c05c08a9f0f6c540854cf6c25c 100755
--- a/bin/router
+++ b/bin/router
@@ -49,7 +49,10 @@ if (!config.get('browserid_url')) {
// #1 - Setup health check / heartbeat middleware.
// This is in front of logging on purpose. see issue #537
-heartbeat.setup(app);
+var browserid_url = urlparse(config.get('browserid_url')).validate().normalize().originOnly();
+heartbeat.setup(app, {
+ dependencies: [browserid_url]
+});
// #2 - logging! all requests other than __heartbeat__ are logged
app.use(express.logger({
@@ -119,7 +122,6 @@ wsapi.setup({
}, app);
// Forward all leftover requests to browserid
-var browserid_url = urlparse(config.get('browserid_url')).validate().normalize().originOnly();
app.use(function(req, res, next) {
forward(
browserid_url+req.url, req, res,
diff --git a/lib/heartbeat.js b/lib/heartbeat.js
index 666e77767ac78f4fb8fdb74355a42a4bf870dadc..e5e309fd4c990366e7cb198bb549744e179ea550 100644
--- a/lib/heartbeat.js
+++ b/lib/heartbeat.js
@@ -4,20 +4,74 @@
const
urlparse = require('urlparse'),
-logger = require('./logging.js').logger;
+logger = require('./logging.js').logger,
+url = require('url');
// the path that heartbeats live at
exports.path = '/__heartbeat__';
+const checkTimeout = 5000;
+
// a helper function to set up a heartbeat check
-exports.setup = function(app, cb) {
+exports.setup = function(app, options, cb) {
+ var dependencies = [];
+
+ if (typeof options == 'function') {
+ cb = options;
+ } else if (options && options.dependencies) {
+ dependencies = options.dependencies;
+ }
+ var count = dependencies.length;
+
app.use(function(req, res, next) {
- if (req.method === 'GET' && req.path === exports.path) {
- function ok(yeah) {
- res.writeHead(yeah ? 200 : 500);
- res.write(yeah ? 'ok' : 'not ok');
- res.end();
+ if (req.method !== 'GET' || req.path !== exports.path) {
+ return next();
+ }
+
+ var checked = 0;
+ var query = url.parse(req.url, true).query;
+ var deep = typeof query.deep != 'undefined';
+ var notOk = [];
+
+ // callback for checking a dependency
+ function checkCB (num) {
+ return function (err, isOk) {
+ checked++;
+ if (err) {
+ notOk.push(dependencies[num] + ': '+ err);
+ }
+
+ // if all dependencies have been checked
+ if (checked == count) {
+ if (notOk.length === 0) {
+ try {
+ if (cb) cb(ok);
+ else ok(true);
+ } catch(e) {
+ logger.error("Exception caught in heartbeat handler: " + e.toString());
+ ok(false, e);
+ }
+ } else {
+ logger.warn("heartbeat failed due to dependencies - " + notOk.join(', '));
+ ok(false, '\n' + notOk.join('\n') + '\n');
+ }
+ }
+ };
+ }
+
+ function ok(yeah, msg) {
+ res.writeHead(yeah ? 200 : 500);
+ res.write(yeah ? 'ok' : 'bad');
+ if (msg) res.write(msg);
+ res.end();
+ }
+
+ // check all dependencies if deep
+ if (deep && count) {
+ for (var i = 0; i < count; i++) {
+ check(dependencies[i] + exports.path, checkCB(i));
}
+ } else {
try {
if (cb) cb(ok);
else ok(true);
@@ -25,29 +79,39 @@ exports.setup = function(app, cb) {
logger.error("Exception caught in heartbeat handler: " + e.toString());
ok(false);
}
- } else {
- return next();
}
});
};
+
// a function to check the heartbeat of a remote server
-exports.check = function(url, cb) {
+var check = exports.check = function(url, cb) {
if (typeof url === 'string') url = urlparse(url).normalize().validate();
else if (typeof url !== 'object') throw "url string or object required as argumnet to heartbeat.check";
if (!url.port) url.port = (url.scheme === 'http') ? 80 : 443;
var shortname = url.host + ':' + url.port;
- require(url.scheme).get({
+ var timeoutHandle = setTimeout(function() {
+ req.abort();
+ }, checkTimeout);
+
+ var req = require(url.scheme).get({
host: url.host,
port: url.port,
path: exports.path
}, function (res) {
- if (res.statusCode === 200) cb(true);
- else logger.error("non-200 response from " + shortname + ". fatal! (" + res.statusCode + ")");
- }, function (e) {
- logger.error("can't communicate with " + shortname + ". fatal: " + e);
- cb(false);
+ clearTimeout(timeoutHandle);
+ if (res.statusCode === 200) cb(null, true);
+ else {
+ logger.warn("heartbeat failure: non-200 response from " + shortname + ". fatal! (" +
+ res.statusCode + ")");
+ cb("response code " + res.statusCode);
+ }
+ });
+ req.on('error', function (e) {
+ clearTimeout(timeoutHandle);
+ logger.warn("heartbeat failure: can't communicate with " + shortname + ". fatal: " + e);
+ cb(e ? e : "unknown error");
});
};
diff --git a/tests/heartbeat-test.js b/tests/heartbeat-test.js
new file mode 100755
index 0000000000000000000000000000000000000000..815f97120003b062da83fe9d898acc0fc16ff471
--- /dev/null
+++ b/tests/heartbeat-test.js
@@ -0,0 +1,122 @@
+#!/usr/bin/env node
+
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+require('./lib/test_env.js');
+
+const assert =
+require('assert'),
+vows = require('vows'),
+start_stop = require('./lib/start-stop.js'),
+wsapi = require('./lib/wsapi.js'),
+db = require('../lib/db.js'),
+config = require('../lib/configuration.js'),
+bcrypt = require('bcrypt'),
+http = require('http');
+
+var suite = vows.describe('heartbeat');
+
+// disable vows (often flakey?) async error behavior
+suite.options.error = false;
+
+start_stop.addStartupBatches(suite);
+
+// test deep and shallow heartbeats work for all processes
+[ 10004, 10002, 10003, 10004, 10007 ].forEach(function(port) {
+ [ true, false ].forEach(function(shallow) {
+ var testName = "shallow heartbeat check for 127.0.0.1:" + port;
+ suite.addBatch({
+ testName: {
+ topic: function() {
+ var self = this;
+
+ var req = http.get({
+ host: '127.0.0.1',
+ port: port,
+ path: '/__heartbeat__' + ( shallow ? "" : "?deep=true")
+ }, function(res) {
+ self.callback(null, res.statusCode);
+ req.abort();
+ }).on('error', function(e) {
+ self.callback(e, null);
+ req.abort();
+ });
+ },
+ "works": function(err, code) {
+ assert.strictEqual(err, null);
+ assert.equal(code, 200);
+ }
+ }
+ });
+ });
+});
+
+// now let's SIGSTOP the browserid process and verify that the router's
+// deep heartbeat fails within 11s
+suite.addBatch({
+ "stopping the browserid process": {
+ topic: function() {
+ process.kill(parseInt(process.env['BROWSERID_PID'], 10), 'SIGSTOP');
+ this.callback();
+ },
+ "then doing a deep __heartbeat__ on router": {
+ topic: function() {
+ var self = this;
+ var start = new Date();
+ var req = http.get({
+ host: '127.0.0.1',
+ port: 10002,
+ path: '/__heartbeat__?deep=true'
+ }, function(res) {
+ self.callback(null, res.statusCode, start);
+ req.abort();
+ }).on('error', function(e) {
+ self.callback(e, null);
+ req.abort();
+ });
+ },
+ "fails": function(e, code, start) {
+ assert.ok(!e);
+ assert.strictEqual(500, code);
+ },
+ "takes about 5s": function(e, code, start) {
+ assert.ok(!e);
+ var elapsedMS = new Date() - start;
+ assert.ok(3000 < elapsedMS < 7000);
+ },
+ "but upon SIGCONT": {
+ topic: function(e, code) {
+ process.kill(parseInt(process.env['BROWSERID_PID'], 10), 'SIGCONT');
+ this.callback();
+ },
+ "a deep heartbeat": {
+ topic: function() {
+ var self = this;
+ var req = http.get(
+ { host: '127.0.0.1', port: 10002, path: '/__heartbeat__?deep=true'},
+ function(res) {
+ self.callback(null, res.statusCode);
+ req.abort();
+ }).on('error', function(e) {
+ self.callback(e, null);
+ req.abort();
+ });
+ },
+ "works": function(err, code) {
+ assert.ok(!err);
+ assert.strictEqual(200, code);
+ }
+ }
+ }
+ }
+ }
+});
+
+
+start_stop.addShutdownBatches(suite);
+
+// run or export the suite.
+if (process.argv[1] === __filename) suite.run();
+else suite.export(module);
diff --git a/tests/lib/start-stop.js b/tests/lib/start-stop.js
index b73aa2ea8f4951a1c615788e750259ab93bd7ece..742fb032745d0607052ff7b19de83157dff95e8a 100644
--- a/tests/lib/start-stop.js
+++ b/tests/lib/start-stop.js
@@ -46,10 +46,13 @@ function setupProc(proc) {
}
}
var tokenRegex = new RegExp('token=([A-Za-z0-9]+)$', 'm');
+ var pidRegex = new RegExp('^spawned (\\w+) \\(.*\\) with pid ([0-9]+)$');
if (!sentReady && /^router.*127\.0\.0\.1:10002$/.test(x)) {
exports.browserid.emit('ready');
sentReady = true;
+ } else if (!sentReady && (m = pidRegex.exec(x))) {
+ process.env[m[1].toUpperCase() + "_PID"] = m[2];
} else if (m = tokenRegex.exec(x)) {
if (!(/forwarding request:/.test(x))) {
tokenStack.push(m[1]);