diff --git a/bin/load_gen b/bin/load_gen index 5eb1ea00b2d4b9b70a8c4c4e30fc0ad9bd74ba62..c8ad67f961170e98b5ca9adc618550c9489a2b38 100755 --- a/bin/load_gen +++ b/bin/load_gen @@ -231,18 +231,27 @@ function poll() { if (activitiesToRun.indexOf(act) !== -1) { outstanding[act]++; activity[act].startFunc(configuration, function(err) { - if (err) winston.error(err); outstanding[act]--; - if (undefined === completed[act]) completed[act] = [ 0, 0 ]; - completed[act][err ? 1 : 0]++; + if (undefined === completed[act]) completed[act] = [ 0, 0, 0 ]; + if (err) { + if (err.indexOf('server is too busy') != -1) { + completed[act][2]++; + } else { + completed[act][1]++; + winston.error(err); + } + } else { + completed[act][0]++; + } }); } else { - if (undefined === completed[act]) completed[act] = [ 0, 0 ]; + if (undefined === completed[act]) completed[act] = [ 0, 0, 0 ]; completed[act][0]++; } } var numErrors = 0; + var num503s = 0; var numStarted = 0; function updateAverages(elapsed) { @@ -252,6 +261,7 @@ function poll() { Object.keys(completed).forEach(function(k) { numActCompleted += completed[k][0]; numErrors += completed[k][1]; + num503s += completed[k][2]; }); completed = { }; var avgUsersThisPeriod = (numActCompleted / activitiesPerUserPerSecond) * (elapsed / 1000); @@ -279,7 +289,8 @@ function poll() { "\t", averages[1].toFixed(2), "\t", averages[2].toFixed(2), "\t", actSumString, - "\t", numErrors ? "(" + numErrors + " ERRORS!)" : ""); + "\t", numErrors ? "(" + numErrors + " ERRORS!)" : "", + "\t", num503s ? " (" + num503s + " 503s)" : ""); } // ** how much time has elapsed since the last poll? diff --git a/lib/bcrypt.js b/lib/bcrypt.js index a9c679b611a0f6507146362b3d9236fa2dde92b3..0c613b14770467a2d1d0062307b93867c0aee033 100644 --- a/lib/bcrypt.js +++ b/lib/bcrypt.js @@ -1,11 +1,13 @@ const computecluster = require('compute-cluster'), logger = require('../lib/logging.js').logger, -bcrypt = require('bcrypt'); +bcrypt = require('bcrypt'), +config = require('./configuration.js'); var cc = new computecluster({ module: path.join(__dirname, "bcrypt-compute.js"), - max_backlog: 100000 + max_backlog: 100000, + max_request_time: config.get('max_compute_duration') }); cc.on('error', function(e) { diff --git a/lib/configuration.js b/lib/configuration.js index 4fb0d277b3837cd2333e443d7a7f8cbcf1983add..92921fcdcf6bdf6fc5ab9f66bb62f21f4f09696e 100644 --- a/lib/configuration.js +++ b/lib/configuration.js @@ -105,8 +105,10 @@ g_configs.production = { authentication_duration_ms: (2 * 7 * 24 * 60 * 60 * 1000), certificate_validity_ms: (24 * 60 * 60 * 1000), min_time_between_emails_ms: (60 * 1000), - // may be specified to manipulate the maximum number of compute - max_compute_processes: undefined + // may be specified to manipulate the maximum number of compute processes + max_compute_processes: undefined, + // return a 503 if a compute process would take over 10s to complete + max_compute_duration: 10 }; @@ -124,7 +126,8 @@ g_configs.local = { authentication_duration_ms: g_configs.production.authentication_duration_ms, certificate_validity_ms: g_configs.production.certificate_validity_ms, min_time_between_emails_ms: g_configs.production.min_time_between_emails_ms, - max_compute_processes: undefined + max_compute_processes: undefined, + max_compute_duration: 10 }; // test environments are variations on local diff --git a/lib/load_gen/activities/add_email.js b/lib/load_gen/activities/add_email.js index d059fdba70f8e7d5d548cb5744729720ccaa1795..3e5e34e5048291f0bbeba8b36c930506b9a8086e 100644 --- a/lib/load_gen/activities/add_email.js +++ b/lib/load_gen/activities/add_email.js @@ -67,6 +67,7 @@ exports.startFunc = function(cfg, cb) { cb = (function() { var _cb = cb; return function(x) { + if (x) userdb.removeLastEmailFromUser(user); userdb.releaseUser(user); _cb(x); }; diff --git a/lib/load_gen/user_db.js b/lib/load_gen/user_db.js index c6ceff78b31ec0cf77cb938bdb991415772048a4..3e15e19efc9434f68b7f84469f688bf8397fcaea 100644 --- a/lib/load_gen/user_db.js +++ b/lib/load_gen/user_db.js @@ -138,6 +138,10 @@ exports.addEmailToUser = function(user) { return email; }; +exports.removeLastEmailFromUser = function(user) { + user.emails.pop(); +}; + exports.addKeyToUserCtx = function(ctx, email) { // this is simulated. it will need to be real to apply load to // the verifier, but that in turn will drastically increase the diff --git a/lib/wsapi/authenticate_user.js b/lib/wsapi/authenticate_user.js index 4749c8f5a6b00fe91e5e47694f40394d718e0d92..ae49c96af004adee9768934090837ad4b23c69b4 100644 --- a/lib/wsapi/authenticate_user.js +++ b/lib/wsapi/authenticate_user.js @@ -38,6 +38,11 @@ exports.process = function(req, res) { statsd.timing('bcrypt.compare_time', reqTime); if (err) { + if (err.indexOf('exceeded') != -1) { + logger.warn("max load hit, failing on auth request with 503: " + err); + res.status(503); + return fail("server is too busy"); + } logger.error("error comparing passwords with bcrypt: " + err); return fail("internal password check error"); } else if (!success) { diff --git a/package.json b/package.json index ef87aafb0264575afe3c508ff1748e842da48a99..b5550b9fae65cec5d50a12f9a329059fd9b0edb6 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ , "dependencies": { "JSONSelect": "0.2.1" , "bcrypt": "0.4.1" - , "compute-cluster": "0.0.5" + , "compute-cluster": "0.0.6" , "connect": "1.7.2" , "connect-cookie-session" : "0.0.2" , "connect-logger-statsd": "0.0.1"