diff --git a/lib/browserid/fake_verification.js b/lib/browserid/fake_verification.js
index 9d1df8bb6ce20a8a76ac5f841d3fecd4db05cfb1..03f33f1c15af0fbeb0f68cd43ce6f0f3e782bed1 100644
--- a/lib/browserid/fake_verification.js
+++ b/lib/browserid/fake_verification.js
@@ -12,7 +12,8 @@ const
 configuration = require('../configuration.js'),
 url = require('url'),
 db = require('../db.js');
-logger = require('../logging.js').logger;
+logger = require('../logging.js').logger,
+wsapi = require('../wsapi');
 
 logger.warn("HEAR YE: Fake verfication enabled, aceess via /wsapi/fake_verification?email=foo@bar.com");
 logger.warn("THIS IS NEVER OK IN A PRODUCTION ENVIRONMENT");
@@ -20,7 +21,8 @@ logger.warn("THIS IS NEVER OK IN A PRODUCTION ENVIRONMENT");
 exports.addVerificationWSAPI = function(app) {
   app.get('/wsapi/fake_verification', function(req, res) {
     var email = url.parse(req.url, true).query['email'];
-    db.verificationSecretForEmail(email, function(secret) {
+    db.verificationSecretForEmail(email, function(err, secret) {
+      if (err) return wsapi.databaseDown(resp, err);
       if (secret) res.write(secret);
       else res.writeHead(400, {"Content-Type": "text/plain"});
       res.end();
diff --git a/lib/configuration.js b/lib/configuration.js
index 78eead9e9a37dfb31eff37c96b9fdc452f775e46..a4fc0bed6f9418ff7bd46647a366ece84dfb9874 100644
--- a/lib/configuration.js
+++ b/lib/configuration.js
@@ -84,7 +84,16 @@ var conf = module.exports = convict({
       env: 'DATABASE_NAME'
     },
     password: 'string?',
-    host: 'string?'
+    host: 'string?',
+    max_query_time_ms: {
+      format: 'integer = 5000',
+      doc: "The maximum amount of time we'll allow a query to run before considering the database to be sick",
+      env: 'MAX_QUERY_TIME_MS'
+    },
+    max_reconnect_attempts: {
+      format: 'integer = 1',
+      doc: "The maximum number of times we'll attempt to reconnect to the database before failing all outstanding queries"
+    }
   },
   smtp: {
     host: 'string?',
diff --git a/lib/db.js b/lib/db.js
index c0670cad8e46ccab2ea7c8f92e6d76e2dd9ab5cf..6765c28fbc71ad132d2c3e5dd69352954c2e7b50 100644
--- a/lib/db.js
+++ b/lib/db.js
@@ -39,7 +39,7 @@ exports.open = function(cfg, cb) {
       ready = true;
       waiting.forEach(function(f) { f() });
       waiting = [];
-      if (cb) cb();
+      if (cb) cb(null);
     }
   });
 };
diff --git a/lib/db/json.js b/lib/db/json.js
index 266ae22b3d29e14993defe99528e447d3d6af7e8..b9a4c64088258d22a75452e88521af511239dc28 100644
--- a/lib/db/json.js
+++ b/lib/db/json.js
@@ -81,41 +81,40 @@ exports.open = function(cfg, cb) {
   logger.debug("opening JSON database: " + dbPath);
 
   sync();
-
-  setTimeout(cb, 0);
+  process.nextTick(function() { cb(null); });
 };
 
 exports.closeAndRemove = function(cb) {
   // if the file cannot be removed, it's not an error, just means it was never
   // written or deleted by a different process
   try { fs.unlinkSync(dbPath); } catch(e) { }
-  setTimeout(function() { cb(undefined); }, 0);
+  process.nextTick(function() { cb(null); });
 };
 
 exports.close = function(cb) {
   // don't flush database here to disk, the database is flushed synchronously when
   // written - If we were to flush here we could overwrite changes made by
   // another process - see issue #557
-  setTimeout(function() { cb(undefined) }, 0);
+  process.nextTick(function() { cb(null) });
 };
 
 exports.emailKnown = function(email, cb) {
   sync();
   var m = jsel.match(".emails ." + ESC(email), db.users);
-  setTimeout(function() { cb(m.length > 0) }, 0);
+  process.nextTick(function() { cb(null, m.length > 0) });
 };
 
 exports.emailType = function(email, cb) {
   sync();
   var m = jsel.match(".emails ." + ESC(email), db.users);
-  process.nextTick(function() { cb(m.length ? m[0].type : undefined); });
+  process.nextTick(function() { cb(null, m.length ? m[0].type : undefined); });
 };
 
 exports.isStaged = function(email, cb) {
   if (cb) {
     setTimeout(function() {
       sync();
-      cb(db.stagedEmails.hasOwnProperty(email));
+      cb(null, db.stagedEmails.hasOwnProperty(email));
     }, 0);
   }
 };
@@ -127,7 +126,7 @@ exports.lastStaged = function(email, cb) {
     if (db.stagedEmails.hasOwnProperty(email)) {
       d = new Date(db.staged[db.stagedEmails[email]].when);
     }
-    setTimeout(function() { cb(d); }, 0);
+    setTimeout(function() { cb(null, d); }, 0);
   }
 };
 
@@ -135,7 +134,7 @@ exports.emailsBelongToSameAccount = function(lhs, rhs, cb) {
   sync();
   var m = jsel.match(".emails:has(."+ESC(lhs)+"):has(."+ESC(rhs)+")", db.users);
   process.nextTick(function() {
-    cb(m && m.length == 1);
+    cb(null, m && m.length == 1);
   });
 };
 
@@ -145,7 +144,7 @@ exports.emailToUID = function(email, cb) {
   if (m.length === 0) m = undefined;
   else m = m[0];
   process.nextTick(function() {
-    cb(m);
+    cb(null, m);
   });
 };
 
@@ -153,7 +152,7 @@ exports.userOwnsEmail = function(uid, email, cb) {
   sync();
   var m = jsel.match(":root > object:has(:root > .id:expr(x=" + ESC(uid) + ")):has(.emails > ." + ESC(email) + ")", db.users);
   process.nextTick(function() {
-    cb(m && m.length == 1);
+    cb(null, m && m.length == 1);
   });
 };
 
@@ -172,7 +171,7 @@ function addEmailToAccount(userID, email, type, cb) {
       emails[0][email] = { type: type };
       flush();
     }
-    cb();
+    cb(null);
   });
 }
 
@@ -187,7 +186,7 @@ exports.stageUser = function(email, cb) {
     };
     db.stagedEmails[email] = secret;
     flush();
-    setTimeout(function() { cb(secret); }, 0);
+    process.nextTick(function() { cb(null, secret); });
   });
 };
 
@@ -204,7 +203,7 @@ exports.stageEmail = function(existing_user, new_email, cb) {
     db.stagedEmails[new_email] = secret;
     flush();
 
-    setTimeout(function() { cb(secret); }, 0);
+    process.nextTick(function() { cb(null, secret); });
   });
 };
 
@@ -219,14 +218,14 @@ exports.createUserWithPrimaryEmail = function(email, cb) {
   });
   flush();
   process.nextTick(function() {
-    cb(undefined, uid);
+    cb(null, uid);
   });
 };
 
 exports.haveVerificationSecret = function(secret, cb) {
   process.nextTick(function() {
     sync();
-    cb(!!(db.staged[secret]));
+    cb(null, !!(db.staged[secret]));
   });
 };
 
@@ -235,8 +234,8 @@ exports.emailForVerificationSecret = function(secret, cb) {
   process.nextTick(function() {
     sync();
     if (!db.staged[secret]) return cb("no such secret");
-    exports.checkAuth(db.staged[secret].existing_user, function (hash) {
-      cb(undefined, {
+    exports.checkAuth(db.staged[secret].existing_user, function (err, hash) {
+      cb(err, {
         email: db.staged[secret].email,
         needs_password: !hash
       });
@@ -247,7 +246,7 @@ exports.emailForVerificationSecret = function(secret, cb) {
 exports.verificationSecretForEmail = function(email, cb) {
   setTimeout(function() {
     sync();
-    cb(db.stagedEmails[email]);
+    cb(null, db.stagedEmails[email]);
   }, 0);
 };
 
@@ -261,7 +260,7 @@ exports.gotVerificationSecret = function(secret, hash, cb) {
   delete db.stagedEmails[o.email];
   flush();
   if (o.type === 'add_account') {
-    exports.emailKnown(o.email, function(known) {
+    exports.emailKnown(o.email, function(err, known) {
       function createAccount() {
         var emailVal = {};
         emailVal[o.email] = { type: 'secondary' };
@@ -272,7 +271,7 @@ exports.gotVerificationSecret = function(secret, hash, cb) {
           emails: emailVal
         });
         flush();
-        cb(undefined, o.email, uid);
+        cb(null, o.email, uid);
       }
 
       // if this email address is known and a user has completed a re-verification of this email
@@ -291,7 +290,7 @@ exports.gotVerificationSecret = function(secret, hash, cb) {
       }
     });
   } else if (o.type === 'add_email') {
-    exports.emailKnown(o.email, function(known) {
+    exports.emailKnown(o.email, function(err, known) {
       function addIt() {
         addEmailToAccount(o.existing_user, o.email, 'secondary', function(e) {
           cb(e, o.email, o.existing_user);
@@ -313,7 +312,7 @@ exports.gotVerificationSecret = function(secret, hash, cb) {
 
 exports.addPrimaryEmailToAccount = function(userID, emailToAdd, cb) {
   sync();
-  exports.emailKnown(emailToAdd, function(known) {
+  exports.emailKnown(emailToAdd, function(err, known) {
     function addIt() {
       addEmailToAccount(userID, emailToAdd, 'primary', cb);
     }
@@ -336,7 +335,7 @@ exports.checkAuth = function(userID, cb) {
     if (m.length === 0) m = undefined;
     else m = m[0];
   }
-  process.nextTick(function() { cb(m) });
+  process.nextTick(function() { cb(null, m) });
 };
 
 exports.userKnown = function(userID, cb) {
@@ -344,7 +343,7 @@ exports.userKnown = function(userID, cb) {
   var m = jsel.match(":root > object:has(:root > .id:expr(x=" + ESC(userID) + "))", db.users);
   if (m.length === 0) m = undefined;
   else m = m[0];
-  process.nextTick(function() { cb(m) });
+  process.nextTick(function() { cb(null, m) });
 };
 
 exports.updatePassword = function(userID, hash, cb) {
@@ -378,7 +377,7 @@ exports.removeEmail = function(authenticated_user, email, cb) {
     delete emails[email];
     flush();
   }
-  setTimeout(function() { cb(); }, 0);
+  setTimeout(function() { cb(null); }, 0);
 };
 
 function removeEmailNoCheck(email, cb) {
@@ -389,7 +388,7 @@ function removeEmailNoCheck(email, cb) {
     delete emails[email];
     flush();
   }
-  process.nextTick(function() { cb(); });
+  process.nextTick(function() { cb(null); });
 };
 
 exports.cancelAccount = function(authenticated_uid, cb) {
@@ -405,7 +404,7 @@ exports.cancelAccount = function(authenticated_uid, cb) {
     flush();
   }
 
-  process.nextTick(function() { cb(); });
+  process.nextTick(function() { cb(null); });
 };
 
 exports.addTestUser = function(email, hash, cb) {
@@ -419,10 +418,10 @@ exports.addTestUser = function(email, hash, cb) {
       emails: emailVal
     });
     flush();
-    cb();
+    cb(null);
   });
 };
 
 exports.ping = function(cb) {
-  setTimeout(function() { cb(); }, 0);
+  process.nextTick(function() { cb(null); });
 };
diff --git a/lib/db/mysql.js b/lib/db/mysql.js
index 8b8839635b94a59297da75df9651d1457412d4f3..2931097bcde0d45dd58fdeef5265e1c1524a2a6f 100644
--- a/lib/db/mysql.js
+++ b/lib/db/mysql.js
@@ -29,13 +29,34 @@
  */
 
 const
-mysql = require('mysql'),
+mysql = require('./mysql_wrapper.js'),
 secrets = require('../secrets.js'),
 logger = require('../logging.js').logger,
-statsd = require('../statsd');
+conf = require('../configuration.js');
 
 var client = undefined;
 
+// for testing!  when 'STALL_MYSQL_WHEN_PRESENT' is defined in the environment,
+// it causes the driver to simulate stalling whent said file is present
+if (conf.get('env') === 'test_mysql' && process.env['STALL_MYSQL_WHEN_PRESENT']) {
+  logger.debug('database driver will be stalled when file is present: ' +
+               process.env['STALL_MYSQL_WHEN_PRESENT']);
+  const fs = require('fs');
+  fs.watchFile(
+    process.env['STALL_MYSQL_WHEN_PRESENT'],
+    { persistent: false, interval: 1 },
+    function (curr, prev) {
+      // stall the database driver when specified file is present
+      fs.stat(process.env['STALL_MYSQL_WHEN_PRESENT'], function(err, stats) {
+        if (client) {
+          var stall = !(err && err.code === 'ENOENT');
+          logger.debug("database driver is " + (stall ? "stalled" : "unblocked"));
+          client.stall(stall);
+        }
+      });
+    });
+}
+
 // If you change these schemas, please notify <services-ops@mozilla.com>
 const schemas = [
   "CREATE TABLE IF NOT EXISTS user (" +
@@ -68,7 +89,7 @@ function logUnexpectedError(detail) {
   var where;
   try { dne; } catch (e) { where = e.stack.split('\n')[2].trim(); };
   // now log it!
-  logger.error("unexpected database failure: " + detail + " -- " + where);
+  logger.warn("unexpected database failure: " + detail + " -- " + where);
 }
 
 // open & create the mysql database
@@ -98,27 +119,6 @@ exports.open = function(cfg, cb) {
     options.database = database;
     client = mysql.createClient(options);
 
-    // replace .query with a function that times queries and
-    // logs to statsd
-    var realQuery = client.query;
-    client.query = function() {
-      var startTime = new Date();
-      var client_cb;
-      var new_cb = function() {
-        var reqTime = new Date - startTime;
-        statsd.timing('query_time', reqTime);
-        if (client_cb) client_cb.apply(null, arguments);
-      };
-      var args = Array.prototype.slice.call(arguments);
-      if (typeof args[args.length - 1] === 'function') {
-        client_cb = args[args.length - 1];
-        args[args.length - 1] = new_cb;
-      } else {
-        args.push(new_cb);
-      }
-      realQuery.apply(client, args);
-    };
-
     client.ping(function(err) {
       logger.debug("connection to database " + (err ? ("fails: " + err) : "established"));
       cb(err);
@@ -176,7 +176,7 @@ exports.close = function(cb) {
   client.end(function(err) {
     client = undefined;
     if (err) logUnexpectedError(err);
-    if (cb) cb(err);
+    if (cb) cb(err === undefined ? null : err);
   });
 };
 
@@ -198,8 +198,7 @@ exports.emailKnown = function(email, cb) {
   client.query(
     "SELECT COUNT(*) as N FROM email WHERE address = ?", [ email ],
     function(err, rows) {
-      if (err) logUnexpectedError(err);
-      cb(rows && rows.length > 0 && rows[0].N > 0);
+      cb(err, rows && rows.length > 0 && rows[0].N > 0);
     }
   );
 };
@@ -208,8 +207,7 @@ exports.userKnown = function(uid, cb) {
   client.query(
     "SELECT COUNT(*) as N FROM user WHERE id = ?", [ uid ],
     function(err, rows) {
-      if (err) logUnexpectedError(err);
-      cb(rows && rows.length > 0 && rows[0].N > 0);
+      cb(err, rows && rows.length > 0 && rows[0].N > 0);
     }
   );
 };
@@ -218,8 +216,7 @@ exports.emailType = function(email, cb) {
   client.query(
     "SELECT type FROM email WHERE address = ?", [ email ],
     function(err, rows) {
-      if (err) logUnexpectedError(err);
-      cb((rows && rows.length > 0) ? rows[0].type : undefined);
+      cb(err, (rows && rows.length > 0) ? rows[0].type : undefined);
     }
   );
 }
@@ -228,8 +225,7 @@ exports.isStaged = function(email, cb) {
   client.query(
     "SELECT COUNT(*) as N FROM staged WHERE email = ?", [ email ],
     function(err, rows) {
-      if (err) logUnexpectedError(err);
-      cb(rows && rows.length > 0 && rows[0].N > 0);
+      cb(err, rows && rows.length > 0 && rows[0].N > 0);
     }
   );
 }
@@ -238,9 +234,9 @@ exports.lastStaged = function(email, cb) {
   client.query(
     "SELECT UNIX_TIMESTAMP(ts) as ts FROM staged WHERE email = ?", [ email ],
     function(err, rows) {
-      if (err) logUnexpectedError(err);
-      if (!rows || rows.length === 0) cb();
-      else cb(new Date(rows[0].ts * 1000));
+      if (err) cb(err);
+      else if (!rows || rows.length === 0) cb(null);
+      else cb(null, new Date(rows[0].ts * 1000));
     }
   );
 };
@@ -252,10 +248,7 @@ exports.stageUser = function(email, cb) {
                  'ON DUPLICATE KEY UPDATE secret=?, existing_user=NULL, new_acct=TRUE, ts=NOW()',
                  [ secret, email, secret],
                  function(err) {
-                   if (err) {
-                     logUnexpectedError(err);
-                     cb(undefined, err);
-                   } else cb(secret);
+                   cb(err, err ? undefined : secret);
                  });
   });
 };
@@ -265,8 +258,7 @@ exports.haveVerificationSecret = function(secret, cb) {
   client.query(
     "SELECT count(*) as n FROM staged WHERE secret = ?", [ secret ],
     function(err, rows) {
-      if (err) cb(false);
-      else cb(rows.length === 1 && rows[0].n === 1);
+      cb(err, rows && rows.length === 1 && rows[0].n === 1);
     });
 };
 
@@ -274,14 +266,15 @@ exports.emailForVerificationSecret = function(secret, cb) {
   client.query(
     "SELECT * FROM staged WHERE secret = ?", [ secret ],
     function(err, rows) {
-      if (err) logUnexpectedError(err);
+      if (err) return cb("database unavailable");
+
       // if the record was not found, fail out
       if (!rows || rows.length != 1) return cb("no such secret");
 
       var o = rows[0];
 
       // if the record was found and this is for a new_acct, return the email
-      if (o.new_acct) return cb(undefined, { email: o.email, needs_password: false });
+      if (o.new_acct) return cb(null, { email: o.email, needs_password: false });
 
       // we need a userid.  the old schema had an 'existing' field which was an email
       // address.  the new schema has an 'existing_user' field which is a userid.
@@ -289,8 +282,8 @@ exports.emailForVerificationSecret = function(secret, cb) {
       // and can be removed in feb 2012 some time.  maybe for valentines day?
       if (typeof o.existing_user === 'number') doCheckAuth(o.existing_user);
       else if (typeof o.existing === 'string') {
-        exports.emailToUID(o.existing, function(uid) {
-          if (uid === undefined) return cb('acct associated with staged email doesn\'t exist');
+        exports.emailToUID(o.existing, function(err, uid) {
+          if (err || uid === undefined) return cb('acct associated with staged email doesn\'t exist');
           doCheckAuth(uid);
         });
       }
@@ -301,8 +294,8 @@ exports.emailForVerificationSecret = function(secret, cb) {
         // are associated with the acct at the moment, then there will not be a
         // password set and the user will need to set one with the addition of
         // this addresss)
-        exports.checkAuth(uid, function(hash) {
-          cb(undefined, {
+        exports.checkAuth(uid, function(err, hash) {
+          cb(err, {
             email: o.email,
             needs_password: !hash
           });
@@ -315,8 +308,7 @@ exports.verificationSecretForEmail = function(email, cb) {
   client.query(
     "SELECT secret FROM staged WHERE email = ?", [ email ],
     function(err, rows) {
-      if (err) logUnexpectedError(err);
-      cb((rows && rows.length > 0) ? rows[0].secret : undefined);
+      cb(err, (rows && rows.length > 0) ? rows[0].secret : undefined);
     });
 };
 
@@ -329,14 +321,14 @@ function addEmailToUser(userID, email, type, cb) {
     "DELETE FROM email WHERE address = ?",
     [ email ],
     function(err, info) {
-      if (err) { logUnexpectedError(err); cb(err); return; }
+      if (err) return cb(err);
       else {
         client.query(
           "INSERT INTO email(user, address, type) VALUES(?, ?, ?)",
           [ userID, email, type ],
           function(err, info) {
             if (err) logUnexpectedError(err);
-            cb(err ? err : undefined, email, userID);
+            cb(err, email, userID);
           });
       }
     });
@@ -363,7 +355,7 @@ exports.gotVerificationSecret = function(secret, hash, cb) {
             "INSERT INTO user(passwd) VALUES(?)",
             [ hash ],
             function(err, info) {
-              if (err) { logUnexpectedError(err); cb(err); return; }
+              if (err) return cb(err);
               addEmailToUser(info.insertId, o.email, 'secondary', cb);
             });
         } else {
@@ -374,7 +366,7 @@ exports.gotVerificationSecret = function(secret, hash, cb) {
           if (typeof o.existing_user === 'number') doAddEmail(o.existing_user);
           else if (typeof o.existing === 'string') {
             exports.emailToUID(o.existing, function(uid) {
-              if (uid === undefined) return cb('acct associated with staged email doesn\'t exist');
+              if (err || uid === undefined) return cb('acct associated with staged email doesn\'t exist');
               doAddEmail(uid);
             });
           }
@@ -400,14 +392,13 @@ exports.createUserWithPrimaryEmail = function(email, cb) {
   client.query(
     "INSERT INTO user() VALUES()",
     function(err, info) {
-      if (err) { logUnexpectedError(err); cb(err); return; }
+      if (err) return cb(err);
       var uid = info.insertId;
       client.query(
         "INSERT INTO email(user, address, type) VALUES(?, ?, ?)",
         [ uid, email, 'primary' ],
         function(err, info) {
-          if (err) logUnexpectedError(err);
-          cb(err ? err : undefined, uid);
+          cb(err, uid);
         });
     });
 };
@@ -417,8 +408,7 @@ exports.emailsBelongToSameAccount = function(lhs, rhs, cb) {
     'SELECT COUNT(*) AS n FROM email WHERE address = ? AND user = ( SELECT user FROM email WHERE address = ? );',
     [ lhs, rhs ],
     function (err, rows) {
-      if (err) cb(false);
-      else cb(rows.length === 1 && rows[0].n === 1);
+      cb(err, rows && rows.length === 1 && rows[0].n === 1);
     });
 }
 
@@ -427,8 +417,7 @@ exports.userOwnsEmail = function(uid, email, cb) {
     'SELECT COUNT(*) AS n FROM email WHERE address = ? AND user = ?',
     [ email, uid ],
     function (err, rows) {
-      if (err) cb(false);
-      else cb(rows.length === 1 && rows[0].n === 1);
+      cb(err, rows && rows.length === 1 && rows[0].n === 1);
     });
 }
 
@@ -439,11 +428,7 @@ exports.stageEmail = function(existing_user, new_email, cb) {
                  'ON DUPLICATE KEY UPDATE secret=?, existing_user=?, new_acct=FALSE, ts=NOW()',
                  [ secret, existing_user, new_email, secret, existing_user],
                  function(err) {
-                   if (err) {
-                     logUnexpectedError(err);
-                     cb(undefined, err);
-                   }
-                   else cb(secret);
+                   cb(err, err ? undefined : secret);
                  });
   });
 };
@@ -453,8 +438,7 @@ exports.emailToUID = function(email, cb) {
     'SELECT user FROM email WHERE address = ?',
     [ email ],
     function (err, rows) {
-      if (err) logUnexpectedError(err);
-      cb((rows && rows.length == 1) ? rows[0].user : undefined);
+      cb(err, (rows && rows.length == 1) ? rows[0].user : undefined);
     });
 };
 
@@ -463,8 +447,7 @@ exports.checkAuth = function(uid, cb) {
     'SELECT passwd FROM user WHERE id = ?',
     [ uid ],
     function (err, rows) {
-      if (err) logUnexpectedError(err);
-      cb((rows && rows.length == 1) ? rows[0].passwd : undefined);
+      cb(err, (rows && rows.length == 1) ? rows[0].passwd : undefined);
     });
 }
 
@@ -473,8 +456,10 @@ exports.updatePassword = function(uid, hash, cb) {
     'UPDATE user SET passwd = ? WHERE id = ?',
     [ hash, uid ],
     function (err, rows) {
-      if (err) logUnexpectedError(err);
-      cb((err || rows.affectedRows !== 1) ? ("no record with email " + email) : undefined);
+      if (!err && (!rows || rows.affectedRows !== 1)) {
+        err = "no record with email " + email;
+      }
+      cb(err);
     });
 }
 
@@ -504,7 +489,9 @@ exports.listEmails = function(uid, cb) {
 };
 
 exports.removeEmail = function(authenticated_user, email, cb) {
-  exports.userOwnsEmail(authenticated_user, email, function(ok) {
+  exports.userOwnsEmail(authenticated_user, email, function(err, ok) {
+    if (err) return cb(err);
+
     if (!ok) {
       logger.warn(authenticated_user + ' attempted to delete an email that doesn\'t belong to her: ' + email);
       cb("authenticated user doesn't have permission to remove specified email " + email);
@@ -515,18 +502,16 @@ exports.removeEmail = function(authenticated_user, email, cb) {
       'DELETE FROM email WHERE address = ?',
       [ email ],
       function(err, info) {
-        if (err) logUnexpectedError(err);
-        // smash null into undefined
-        cb(err ? err : undefined);
+        cb(err);
       });
   });
 };
 
 exports.cancelAccount = function(uid, cb) {
-  function reportErr(err) { if (err) logUnexpectedError(err); }
-  client.query("DELETE LOW_PRIORITY FROM email WHERE user = ?", [ uid ], reportErr);
-  client.query("DELETE LOW_PRIORITY FROM user WHERE id = ?", [ uid ], reportErr);
-  process.nextTick(cb);
+  client.query("DELETE LOW_PRIORITY FROM email WHERE user = ?", [ uid ], function(err) {
+    if (err) return cb(err);
+    client.query("DELETE LOW_PRIORITY FROM user WHERE id = ?", [ uid ], cb);
+  });
 };
 
 exports.addTestUser = function(email, hash, cb) {
@@ -534,17 +519,14 @@ exports.addTestUser = function(email, hash, cb) {
     "INSERT INTO user(passwd) VALUES(?)",
     [ hash ],
     function(err, info) {
-      if (err) {
-        logUnexpectedError(err);
-        cb(err);
-        return;
-      }
+      if (err) return cb(err);
+
       client.query(
         "INSERT INTO email(user, address) VALUES(?, ?)",
         [ info.insertId, email ],
         function(err, info) {
           if (err) logUnexpectedError(err);
-          cb(err ? err : undefined, email);
+          cb(err, err ? null : email);
         });
     });
 };
diff --git a/lib/db/mysql_wrapper.js b/lib/db/mysql_wrapper.js
new file mode 100644
index 0000000000000000000000000000000000000000..386813382ee7e4bd181d169aafbd0e8ddafdf5f8
--- /dev/null
+++ b/lib/db/mysql_wrapper.js
@@ -0,0 +1,131 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This abstraction wraps the mysql driver and provides application level
+ * queueing, as well as query timing and reconnect upon an apparently "stalled"
+ * driver
+ */
+
+const
+mysql = require('mysql'),
+statsd = require('../statsd'),
+logger = require('../logging.js').logger,
+config = require('../configuration.js');
+
+exports.createClient = function(options) {
+  // the application level query queue
+  var queryQueue = [];
+  // The slowQueryTimer is !null when a query is running, and holds
+  // the result from setTimeout.  This variable is both a means to
+  // check if a query is running (only one runs at a time), and as
+  // the timeout handle.
+  var slowQueryTimer = null;
+  // how many consecutive failures have we seen when running queries?
+  var consecutiveFailures = 0;
+  // a testing feature.  By calling `client.stall` you can
+  // cause responses to be dropped which will trigger slow query detection
+  var stalled = false;
+
+  var client = {
+    stall: function(stalledState) {
+      stalled = stalledState;
+    },
+    realClient: null,
+    _resetConnection: function() {
+      if (this.realClient) this.realClient.destroy();
+      this.realClient = mysql.createClient(options);
+      this.realClient.on('error', function(e) {
+        logger.warn("database connection down: " + e.toString());
+      });
+    },
+    ping: function(cb) {
+      this.realClient.ping(cb);
+    },
+    _runNextQuery: function() {
+      var self = this;
+
+      if (slowQueryTimer !== null || !queryQueue.length) return;
+
+      var work = queryQueue.shift();
+
+      function invokeCallback(cb, err, rez) {
+        if (cb) {
+          process.nextTick(function() {
+            try {
+              cb(err, rez);
+            } catch(e) {
+              logger.error('database query callback failed: ' + e.toString());
+            }
+          });
+        }
+      }
+
+      slowQueryTimer = setTimeout(function() {
+        if (++consecutiveFailures > config.get('database.max_reconnect_attempts')) {
+          // if we can't run the query multiple times in a row, we'll fail all outstanding
+          // queries, and reinitialize the connection, so that the process stays up and
+          // retries mysql connection the next time a request which requires db interaction
+          // comes in.
+          queryQueue.unshift(work);
+          logger.warn("cannot reconnect to mysql! " + queryQueue.length + " outstanding queries #fail.");
+          queryQueue.forEach(function(work) {
+            invokeCallback(work.cb, "database connection unavailable");
+          });
+          queryQueue = [];
+          self._resetConnection();
+          slowQueryTimer = null;
+        } else {
+          logger.warn("Query taking more than " + config.get('database.max_query_time_ms') + "ms!  reconnecting to mysql");
+          queryQueue.unshift(work);
+          self._resetConnection();
+          slowQueryTimer = null;
+          self._runNextQuery();
+        }
+      }, config.get('database.max_query_time_ms'));
+
+      this.realClient.query(work.query, work.args, function(err, r) {
+        // if we want to simulate a "stalled" mysql connection, we simply
+        // ignore the results from a query.
+        if (stalled) return;
+
+        clearTimeout(slowQueryTimer);
+        slowQueryTimer = null;
+        consecutiveFailures = 0;
+
+        var reqTime = new Date - work.startTime;
+        statsd.timing('query_time', reqTime);
+
+        invokeCallback(work.cb, err, r);
+        self._runNextQuery();
+      });
+    },
+    query: function() {
+      var client_cb;
+      var args = Array.prototype.slice.call(arguments);
+      var query = args.shift();
+      if (args.length && typeof args[args.length - 1] === 'function') {
+        client_cb = args.pop();
+      }
+      args = args.length ? args[0] : [];
+      queryQueue.push({
+        query: query,
+        args: args,
+        cb: client_cb,
+        // record the time .query was called by the application for
+        // true end to end query timing in statsd
+        startTime: new Date()
+      });
+      this._runNextQuery();
+    },
+    end: function(cb) {
+      this.realClient.end(cb);
+    },
+    useDatabase: function(db, cb) {
+      this.realClient.useDatabase(db, cb);
+    }
+  };
+  client._resetConnection();
+  client.database = client.realClient.database;
+  return client;
+};
diff --git a/lib/httputils.js b/lib/httputils.js
index e34019dc3478fc7253528d5de64876d68651f346..81e68334d52bcf82484bef7e0d086f8ab471e17f 100644
--- a/lib/httputils.js
+++ b/lib/httputils.js
@@ -24,6 +24,10 @@ exports.serverError = function(resp, reason) {
   sendResponse(resp, "Server Error", reason, 500);
 };
 
+exports.serviceUnavailable = function(resp, reason) {
+  sendResponse(resp, "Service Unavailable", reason, 503);
+};
+
 exports.badRequest = function(resp, reason) {
   sendResponse(resp, "Bad Request", reason, 400);
 };
diff --git a/lib/wsapi.js b/lib/wsapi.js
index 4bc001d41cbe85de69c2afa5c28f85ed63f9d5aa..e42d6f828259d12a496d568d35a4088762aab268 100644
--- a/lib/wsapi.js
+++ b/lib/wsapi.js
@@ -107,6 +107,11 @@ function langContext(req) {
   };
 }
 
+function databaseDown(res, err) {
+  logger.warn('database is down, cannot process request: ' + err);
+  httputils.serviceUnavailable(res, "database unavailable");
+}
+
 // common functions exported, for use by different api calls
 exports.clearAuthenticatedUser = clearAuthenticatedUser;
 exports.isAuthed = isAuthed;
@@ -115,6 +120,7 @@ exports.authenticateSession = authenticateSession;
 exports.checkPassword = checkPassword;
 exports.fowardWritesTo = undefined;
 exports.langContext = langContext;
+exports.databaseDown = databaseDown;
 
 exports.setup = function(options, app) {
   const WSAPI_PREFIX = '/wsapi/';
diff --git a/lib/wsapi/account_cancel.js b/lib/wsapi/account_cancel.js
index c91428f8745102980299e4b0bfa1d4b31b59099c..a0e3644ab3183aeda642259f4e40a967886aa449 100644
--- a/lib/wsapi/account_cancel.js
+++ b/lib/wsapi/account_cancel.js
@@ -4,7 +4,7 @@
 
 const
 db = require('../db.js'),
-httputils = require('../httputils'),
+wsapi = require('../wsapi'),
 logger = require('../logging.js').logger;
 
 exports.method = 'post';
@@ -15,8 +15,7 @@ exports.i18n = false;
 exports.process = function(req, res) {
   db.cancelAccount(req.session.userid, function(error) {
     if (error) {
-      logger.error("error canceling account : " + error.toString());
-      httputils.badRequest(res, error.toString());
+      wsapi.databaseDown(res, error);
     } else {
       res.json({ success: true });
     }});
diff --git a/lib/wsapi/add_email_with_assertion.js b/lib/wsapi/add_email_with_assertion.js
index e0ab5a626ce38087d88974cacd306318992a36d3..a86ec6f40662d57427eac6825bf9530bf0089e97 100644
--- a/lib/wsapi/add_email_with_assertion.js
+++ b/lib/wsapi/add_email_with_assertion.js
@@ -24,6 +24,7 @@ exports.i18n = false;
 exports.process = function(req, res) {
   // first let's verify that the assertion is valid
   primary.verifyAssertion(req.body.assertion, function(err, email) {
+    console.log("MOTHERFUCKER", err);
     if (err) {
       return res.json({
         success: false,
@@ -38,10 +39,7 @@ exports.process = function(req, res) {
       if (err) {
         logger.warn('cannot add primary email "' + email + '" to acct with uid "'
                     + req.session.userid + '": ' + err);
-        return res.json({
-          success: false,
-          reason: "database error"
-        });
+        return wsapi.databaseDown(res, err);
       }
 
       // success!
diff --git a/lib/wsapi/address_info.js b/lib/wsapi/address_info.js
index 0cd29bd0595afd5121533bf6adbb89faaec2582d..bfccae4b9af457f43520e46dcfc2aa81de5d67fa 100644
--- a/lib/wsapi/address_info.js
+++ b/lib/wsapi/address_info.js
@@ -4,7 +4,8 @@
 
 const
 db = require('../db.js'),
-primary = require('../primary.js');
+primary = require('../primary.js'),
+wsapi = require('../wsapi.js');
 
 // return information about an email address.
 //   type:  is this an address with 'primary' or 'secondary' support?
@@ -27,28 +28,25 @@ exports.process = function(req, resp) {
   var email = url.parse(req.url, true).query['email'];
   var m = emailRegex.exec(email);
   if (!m) {
-    resp.sendHeader(400);
-    resp.json({ "error": "invalid email address" });
-    return;
+    return httputils.badRequest(resp, "invalid email address");
   }
 
   primary.checkSupport(m[1], function(err, rv) {
     if (err) {
       logger.warn('error checking "' + m[1] + '" for primary support: ' + err);
-      resp.sendHeader(500);
-      resp.json({ "error": "can't check email address" });
-      return;
+      return httputils.serverError(resp, "can't check email address");
     }
 
     if (rv) {
       rv.type = 'primary';
       resp.json(rv);
     } else {
-      db.emailKnown(email, function(known) {
-        resp.json({
-          type: 'secondary',
-          known: known
-        });
+      db.emailKnown(email, function(err, known) {
+        if (err) {
+          return wsapi.databaseDown(resp, err);
+        } else {
+          resp.json({ type: 'secondary', known: known });
+        }
       });
     }
   });
diff --git a/lib/wsapi/auth_with_assertion.js b/lib/wsapi/auth_with_assertion.js
index 3cd5075e9fd7351103cd97d43dada13efc7e5405..8781151358379e93e4f3e0bbe09c182c62b75667 100644
--- a/lib/wsapi/auth_with_assertion.js
+++ b/lib/wsapi/auth_with_assertion.js
@@ -33,10 +33,13 @@ exports.process = function(req, res) {
     }
 
     // 2. if valid, does the user exist?
-    db.emailType(email, function(type) {
+    db.emailType(email, function(err, type) {
+      if (err) return wsapi.databaseDown(res, err);
+
       // if this is a known primary email, authenticate the user and we're done!
       if (type === 'primary') {
-        return db.emailToUID(email, function(uid) {
+        return db.emailToUID(email, function(err, uid) {
+          if (err) return wsapi.databaseDown(res, err);
           if (!uid) return res.json({ success: false, reason: "internal error" });
           wsapi.authenticateSession(req.session, uid, 'assertion');
           return res.json({ success: true });
diff --git a/lib/wsapi/authenticate_user.js b/lib/wsapi/authenticate_user.js
index 22cc195e9360ec46c3455c48c54ad4ec7d1ba405..b1715a1b4c21fce281502e366ab4c1b47b8877fd 100644
--- a/lib/wsapi/authenticate_user.js
+++ b/lib/wsapi/authenticate_user.js
@@ -27,12 +27,16 @@ exports.process = function(req, res) {
     return res.json(r);
   }
 
-  db.emailToUID(req.body.email, function(uid) {
+  db.emailToUID(req.body.email, function(err, uid) {
+    if (err) return wsapi.databaseDown(res, err);
+
     if (typeof uid !== 'number') {
       return fail('no such user');
     }
 
-    db.checkAuth(uid, function(hash) {
+    db.checkAuth(uid, function(err, hash) {
+      if (err) return wsapi.databaseDown(res, err);
+
       if (typeof hash !== 'string') {
         return fail('no password set for user');
       }
diff --git a/lib/wsapi/cert_key.js b/lib/wsapi/cert_key.js
index 84b1b7582dd865174d381eae326723a3661c410a..a6787272ab007369cceb6f8b109b4dc9a64bc85b 100644
--- a/lib/wsapi/cert_key.js
+++ b/lib/wsapi/cert_key.js
@@ -8,7 +8,8 @@ httputils = require('../httputils'),
 logger = require('../logging.js').logger,
 forward = require('../http_forward.js'),
 config = require('../configuration.js'),
-urlparse = require('urlparse');
+urlparse = require('urlparse'),
+wsapi = require('../wsapi.js');
 
 exports.method = 'post';
 exports.writes_db = false;
@@ -17,7 +18,9 @@ exports.args = ['email','pubkey'];
 exports.i18n = false;
 
 exports.process = function(req, res) {
-  db.userOwnsEmail(req.session.userid, req.body.email, function(owned) {
+  db.userOwnsEmail(req.session.userid, req.body.email, function(err, owned) {
+    if (err) return wsapi.databaseDown(res, err);
+
     // not same account? big fat error
     if (!owned) return httputils.badRequest(res, "that email does not belong to you");
 
diff --git a/lib/wsapi/complete_email_addition.js b/lib/wsapi/complete_email_addition.js
index 7c61b60a6c85700c1a623ce8b1dab6ab9b2f5aac..1359b49c379efab649d942f988f37f8c466c7dfa 100644
--- a/lib/wsapi/complete_email_addition.js
+++ b/lib/wsapi/complete_email_addition.js
@@ -21,6 +21,10 @@ exports.process = function(req, res) {
   // is currently NULL - this would occur in the case where this is the
   // first secondary address to be added to an account
   db.emailForVerificationSecret(req.body.token, function(err, r) {
+    if (err === 'database unavailable') {
+      return wsapi.databaseDown(res, err);
+    }
+
     if (!err && r.needs_password && !req.body.pass) {
       err = "user must choose a password";
     }
@@ -40,7 +44,7 @@ exports.process = function(req, res) {
     db.gotVerificationSecret(req.body.token, req.body.pass, function(e, email, uid) {
       if (e) {
         logger.warn("couldn't complete email verification: " + e);
-        res.json({ success: false });
+        wsapi.databaseDown(res, e);
       } else {
         // now do we need to set the password?
         if (r.needs_password && req.body.pass) {
@@ -52,12 +56,13 @@ exports.process = function(req, res) {
             db.updatePassword(uid, hash, function(err) {
               if (err) {
                 logger.warn("couldn't update password during email verification: " + err);
+                wsapi.databaseDown(res, err);
               } else {
-                // XXX: what if our software 503s?  User doens't get a password set and
+                // XXX: what if our software 503s?  User doesn't get a password set and
                 // cannot change it.
                 wsapi.authenticateSession(req.session, uid, 'password');
+                res.json({ success: !err });
               }
-              res.json({ success: !err });
             });
           });
         } else {
diff --git a/lib/wsapi/complete_user_creation.js b/lib/wsapi/complete_user_creation.js
index 5249c90bb96e74b4f48098c8b826768a2444cccd..882351b630f784c34528302de0682ff9870859fc 100644
--- a/lib/wsapi/complete_user_creation.js
+++ b/lib/wsapi/complete_user_creation.js
@@ -26,17 +26,17 @@ exports.process = function(req, res) {
   // We should check to see if the verification secret is valid *before*
   // bcrypting the password (which is expensive), to prevent a possible
   // DoS attack.
-  db.haveVerificationSecret(req.body.token, function(known) {
+  db.haveVerificationSecret(req.body.token, function(err, known) {
+    if (err) return wsapi.databaseDown(res, err);
+
     if (!known) return res.json({ success: false} );
 
     // now bcrypt the password
     wsapi.bcryptPassword(req.body.pass, function (err, hash) {
       if (err) {
-        console.log(err);
         if (err.indexOf('exceeded') != -1) {
           logger.warn("max load hit, failing on auth request with 503: " + err);
-          res.status(503);
-          return res.json({ success: false, reason: "server is too busy" });
+          return httputils.serviceUnavailable("server is too busy");
         }
         logger.error("can't bcrypt: " + err);
         return res.json({ success: false });
@@ -45,7 +45,7 @@ exports.process = function(req, res) {
       db.gotVerificationSecret(req.body.token, hash, function(err, email, uid) {
         if (err) {
           logger.warn("couldn't complete email verification: " + err);
-          res.json({ success: false });
+          wsapi.databaseDown(res, err);
         } else {
           // FIXME: not sure if we want to do this (ba)
           // at this point the user has set a password associated with an email address
diff --git a/lib/wsapi/create_account_with_assertion.js b/lib/wsapi/create_account_with_assertion.js
index 58cf266465c7d636e5eeb55f21202f883a5264bd..13f96d395fb6b56c7f4fa85adfcbc38472ea48cf 100644
--- a/lib/wsapi/create_account_with_assertion.js
+++ b/lib/wsapi/create_account_with_assertion.js
@@ -27,11 +27,7 @@ exports.process = function(req, res) {
     }
 
     db.createUserWithPrimaryEmail(email, function(err, uid) {
-      if (err) {
-        // yikes.  couldn't write database?
-        logger.error('error creating user with primary email address for "'+email+'": ' + err);
-        return httputils.serverError(res);
-      }
+      if (err) return wsapi.databaseDown(res);
       res.json({ success: true, userid: uid });
     });
   });
diff --git a/lib/wsapi/email_addition_status.js b/lib/wsapi/email_addition_status.js
index 833820e39aaf7db6ed4d32f02cb85894ba8639e8..5a7a3017d536a095cb2072137e94a6c6fa436021 100644
--- a/lib/wsapi/email_addition_status.js
+++ b/lib/wsapi/email_addition_status.js
@@ -3,7 +3,8 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 const
-db = require('../db.js');
+db = require('../db.js'),
+wsapi = require('../wsapi.js');
 
 /* First half of account creation.  Stages a user account for creation.
  * this involves creating a secret url that must be delivered to the
@@ -25,15 +26,19 @@ exports.process = function(req, res) {
   db.userOwnsEmail(
     req.session.userid,
     email,
-    function(registered) {
-      if (registered) {
+    function(err, registered) {
+      if (err) {
+        wsapi.databaseDown(res, err);
+      } else if (registered) {
         delete req.session.pendingAddition;
         res.json({ status: 'complete' });
       } else if (!req.session.pendingAddition) {
         res.json({ status: 'failed' });
       } else {
-        db.haveVerificationSecret(req.session.pendingAddition, function (known) {
-          if (known) {
+        db.haveVerificationSecret(req.session.pendingAddition, function (err, known) {
+          if (err) {
+            return wsapi.databaseDown(res, err);
+          } else if (known) {
             return res.json({ status: 'pending' });
           } else {
             delete req.session.pendingAddition;
diff --git a/lib/wsapi/email_for_token.js b/lib/wsapi/email_for_token.js
index 4856cdcf0873a6fa3405134e83324faf419242a4..bfb122a747e8a514d1aca045fb3f14d7567fc6de 100644
--- a/lib/wsapi/email_for_token.js
+++ b/lib/wsapi/email_for_token.js
@@ -3,7 +3,8 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 const
-db = require('../db.js');
+db = require('../db.js'),
+httputils = require('../httputils.js');
 
 /* First half of account creation.  Stages a user account for creation.
  * this involves creating a secret url that must be delivered to the
@@ -20,10 +21,14 @@ exports.i18n = false;
 exports.process = function(req, res) {
   db.emailForVerificationSecret(req.query.token, function(err, r) {
     if (err) {
-      res.json({
-        success: false,
-        reason: err
-      });
+      if (err === 'database unavailable') {
+        httputils.serviceUnavailable(res, err);
+      } else {
+        res.json({
+          success: false,
+          reason: err
+        });
+      }
     } else {
       res.json({
         success: true,
diff --git a/lib/wsapi/have_email.js b/lib/wsapi/have_email.js
index 05b88930b230be51ff30f1f2e18064f1d1ed8400..ec832546bc261fa1970197a5eda4c0e7312afcd0 100644
--- a/lib/wsapi/have_email.js
+++ b/lib/wsapi/have_email.js
@@ -3,7 +3,8 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 const
-db = require('../db.js');
+db = require('../db.js'),
+wsapi = require('../wsapi.js');
 
 // return if an email is known to browserid
 
@@ -15,7 +16,8 @@ exports.i18n = false;
 
 exports.process = function(req, resp) {
   var email = url.parse(req.url, true).query['email'];
-  db.emailKnown(email, function(known) {
+  db.emailKnown(email, function(err, known) {
+    if (err) return wsapi.databaseDown(resp, err);
     resp.json({ email_known: known });
   });
 };
diff --git a/lib/wsapi/list_emails.js b/lib/wsapi/list_emails.js
index fbe4fd64b5fa4bdc9cab607ceccd2cba638c4ca4..6da607007c3b967b53ccc4b6d77834c705db4387 100644
--- a/lib/wsapi/list_emails.js
+++ b/lib/wsapi/list_emails.js
@@ -4,7 +4,8 @@
 
 const
 db = require('../db.js'),
-logger = require('../logging.js').logger;
+logger = require('../logging.js').logger,
+wsapi = require('../wsapi.js');
 
 // returns a list of emails owned by the user:
 //
@@ -21,7 +22,7 @@ exports.i18n = false;
 exports.process = function(req, resp) {
   logger.debug('listing emails for user ' + req.session.userid);
   db.listEmails(req.session.userid, function(err, emails) {
-    if (err) httputils.serverError(resp, err);
+    if (err) wsapi.databaseDown(resp, err);
     else resp.json(emails);
   });
 };
diff --git a/lib/wsapi/remove_email.js b/lib/wsapi/remove_email.js
index fe7dc3c93265dfb2210628604ebe18b7b60a3b84..145adf03deab484899eb96fffacfb55b1d3b0d90 100644
--- a/lib/wsapi/remove_email.js
+++ b/lib/wsapi/remove_email.js
@@ -4,6 +4,7 @@
 
 const
 db = require('../db.js'),
+wsapi = require('../wsapi'),
 httputils = require('../httputils'),
 logger = require('../logging.js').logger;
 
@@ -18,8 +19,12 @@ exports.process = function(req, res) {
 
   db.removeEmail(req.session.userid, email, function(error) {
     if (error) {
-      logger.error("error removing email " + email);
-      httputils.badRequest(res, error.toString());
+      logger.warn("error removing email " + email);
+      if (error === 'database connection unavailable') {
+        wsapi.databaseDown(res, error);
+      } else {
+        httputils.badRequest(res, error.toString());
+      }
     } else {
       res.json({ success: true });
     }});
diff --git a/lib/wsapi/session_context.js b/lib/wsapi/session_context.js
index 08a82cf01e0234808ab7923327bc6f6612087e8a..8b7f9e13d7a058f28192ae354cfe55b2a9e5b09b 100644
--- a/lib/wsapi/session_context.js
+++ b/lib/wsapi/session_context.js
@@ -59,8 +59,10 @@ exports.process = function(req, res) {
     logger.debug("user is not authenticated");
     sendResponse();
   } else {
-    db.userKnown(req.session.userid, function (known) {
-      if (!known) {
+    db.userKnown(req.session.userid, function (err, known) {
+      if (err) {
+        return wsapi.databaseDown(res, err);
+      } else if (!known) {
         logger.debug("user is authenticated with an account that doesn't exist in the database");
         wsapi.clearAuthenticatedUser(req.session);
       } else {
diff --git a/lib/wsapi/stage_email.js b/lib/wsapi/stage_email.js
index 9b4061bc8cba06187a672636e37eaaf85bf4dcdc..8acda357269408aacc0fb5741172dd9015c4709c 100644
--- a/lib/wsapi/stage_email.js
+++ b/lib/wsapi/stage_email.js
@@ -22,7 +22,9 @@ exports.args = ['email','site'];
 exports.i18n = true;
 
 exports.process = function(req, res) {
-  db.lastStaged(req.body.email, function (last) {
+  db.lastStaged(req.body.email, function (err, last) {
+    if (err) return wsapi.databaseDown(res, err);
+
     if (last && (new Date() - last) < config.get('min_time_between_emails_ms')) {
       logger.warn('throttling request to stage email address ' + req.body.email + ', only ' +
                   ((new Date() - last) / 1000.0) + "s elapsed");
@@ -31,7 +33,9 @@ exports.process = function(req, res) {
 
     try {
       // on failure stageEmail may throw
-      db.stageEmail(req.session.userid, req.body.email, function(secret) {
+      db.stageEmail(req.session.userid, req.body.email, function(err, secret) {
+        if (err) return wsapi.databaseDown(res, err);
+
         var langContext = wsapi.langContext(req);
 
         // store the email being added in session data
diff --git a/lib/wsapi/stage_user.js b/lib/wsapi/stage_user.js
index 580838037c2d40eee2c74117eb1f93830fed5f9d..14bb947e148a270e4f5b25c7714a1d24f94114fd 100644
--- a/lib/wsapi/stage_user.js
+++ b/lib/wsapi/stage_user.js
@@ -27,7 +27,9 @@ exports.process = function(req, resp) {
   // staging a user logs you out.
   wsapi.clearAuthenticatedUser(req.session);
 
-  db.lastStaged(req.body.email, function (last) {
+  db.lastStaged(req.body.email, function (err, last) {
+    if (err) return wsapi.databaseDown(resp, err);
+
     if (last && (new Date() - last) < config.get('min_time_between_emails_ms')) {
       logger.warn('throttling request to stage email address ' + req.body.email + ', only ' +
                   ((new Date() - last) / 1000.0) + "s elapsed");
@@ -37,7 +39,9 @@ exports.process = function(req, resp) {
     try {
       // upon success, stage_user returns a secret (that'll get baked into a url
       // and given to the user), on failure it throws
-      db.stageUser(req.body.email, function(secret) {
+      db.stageUser(req.body.email, function(err, secret) {
+        if (err) return wsapi.databaseDown(resp, err);
+
         // store the email being registered in the session data
         if (!req.session) req.session = {};
 
diff --git a/lib/wsapi/update_password.js b/lib/wsapi/update_password.js
index e98a285b055ef56b747c8a7edb6743bffb41a495..d7a395c3a49a7cf4d8330bf30b703443ce0fe3de 100644
--- a/lib/wsapi/update_password.js
+++ b/lib/wsapi/update_password.js
@@ -24,7 +24,9 @@ exports.process = function(req, res) {
     });
   }
 
-  db.checkAuth(req.session.userid, function(hash) {
+  db.checkAuth(req.session.userid, function(err, hash) {
+    if (err) return wsapi.databaseDown(res, err);
+
     if (typeof hash !== 'string' || typeof req.body.oldpass !== 'string')
     {
       return res.json({ success: false });
@@ -62,9 +64,10 @@ exports.process = function(req, res) {
           var success = true;
           if (err) {
             logger.error("error updating bcrypted password for email " + req.body.email, err);
-            success = false;
+            wsapi.databaseDown(res, err);
+          } else {
+            res.json({ success: success });
           }
-          return res.json({ success: success });
         });
       });
     });
diff --git a/lib/wsapi/user_creation_status.js b/lib/wsapi/user_creation_status.js
index e866430862a653f1a4bbd24c425d4db0d434079f..e6812c9c26bd722aef6f17f074b04bffaff1402c 100644
--- a/lib/wsapi/user_creation_status.js
+++ b/lib/wsapi/user_creation_status.js
@@ -17,8 +17,9 @@ exports.process = function(req, res) {
 
   // if the user is authenticated as the user in question, we're done
   if (wsapi.isAuthed(req, 'assertion')) {
-    db.userOwnsEmail(req.session.userid, email, function(owned) {
-      if (owned) res.json({ status: 'complete' });
+    db.userOwnsEmail(req.session.userid, email, function(err, owned) {
+      if (err) wsapi.databaseDown(res, err);
+      else if (owned) res.json({ status: 'complete' });
       else notAuthed();
     });
   } else {
@@ -34,7 +35,9 @@ exports.process = function(req, res) {
 
     // if the secret is still in the database, it hasn't yet been verified and
     // verification is still pending
-    db.haveVerificationSecret(req.session.pendingCreation, function (known) {
+    db.haveVerificationSecret(req.session.pendingCreation, function (err, known) {
+      if (err) return wsapi.databaseDown(res, err);
+
       if (known) return res.json({ status: 'pending' });
       // if the secret isn't known, and we're not authenticated, then the user must authenticate
       // (maybe they verified the URL on a different browser, or maybe they canceled the account
diff --git a/scripts/test_db_connectivity.js b/scripts/test_db_connectivity.js
index 3b1bad4283c7252c0c3f8b99466585097d85c43b..364d44a759b6846e1a3fd3fdfe383353c70ec4ef 100755
--- a/scripts/test_db_connectivity.js
+++ b/scripts/test_db_connectivity.js
@@ -21,8 +21,8 @@ var dbCfg = configuration.get('database');
 // don't bother creating the schema
 delete dbCfg.create_schema;
 
-db.open(dbCfg, function (r) {
-  if (r && r.message === "Unknown database 'browserid'") r = undefined;
+db.open(dbCfg, function (err, r) {
+  if (err && err.message === "Unknown database 'browserid'") r = undefined;
   function end() { process.exit(r === undefined ? 0 : 1); }
   if (r === undefined) db.close(end);
   else end();
diff --git a/tests/db-test.js b/tests/db-test.js
index e25def6e19a64badc16fb1248086f5382b79c62a..65abaffa2926f260fd0157a7ee27dbf9f36d5565 100755
--- a/tests/db-test.js
+++ b/tests/db-test.js
@@ -36,8 +36,8 @@ suite.addBatch({
     topic: function() {
       db.open(dbCfg, this.callback);
     },
-    "and its ready": function(r) {
-      assert.isUndefined(r);
+    "and its ready": function(err) {
+      assert.isNull(err);
     },
     "doesn't prevent onReady": {
       topic: function() { db.onReady(this.callback); },
@@ -54,7 +54,8 @@ suite.addBatch({
     topic: function() {
       db.isStaged('lloyd@nowhe.re', this.callback);
     },
-    "isStaged returns false": function (r) {
+    "isStaged returns false": function (err, r) {
+      assert.isNull(err);
       assert.isFalse(r);
     }
   },
@@ -62,7 +63,8 @@ suite.addBatch({
     topic: function() {
       db.emailKnown('lloyd@nowhe.re', this.callback);
     },
-    "emailKnown returns false": function (r) {
+    "emailKnown returns false": function (err, r) {
+      assert.isNull(err);
       assert.isFalse(r);
     }
   }
@@ -73,13 +75,14 @@ suite.addBatch({
     topic: function() {
       db.stageUser('lloyd@nowhe.re', this.callback);
     },
-    "staging returns a valid secret": function(r) {
+    "staging returns a valid secret": function(err, r) {
+      assert.isNull(err);
       secret = r;
       assert.isString(secret);
       assert.strictEqual(secret.length, 48);
     },
     "fetch email for given secret": {
-      topic: function(secret) {
+      topic: function(err, secret) {
         db.emailForVerificationSecret(secret, this.callback);
       },
       "matches expected email": function(err, r) {
@@ -87,10 +90,11 @@ suite.addBatch({
       }
     },
     "fetch secret for email": {
-      topic: function(secret) {
+      topic: function(err, secret) {
         db.verificationSecretForEmail('lloyd@nowhe.re', this.callback);
       },
-      "matches expected secret": function(storedSecret) {
+      "matches expected secret": function(err, storedSecret) {
+        assert.isNull(err);
         assert.strictEqual(storedSecret, secret);
       }
     }
@@ -102,7 +106,8 @@ suite.addBatch({
     topic: function() {
       db.isStaged('lloyd@nowhe.re', this.callback);
     },
-    " as staged after it is": function (r) {
+    " as staged after it is": function (err, r) {
+      assert.isNull(err);
       assert.strictEqual(r, true);
     }
   },
@@ -110,7 +115,8 @@ suite.addBatch({
     topic: function() {
       db.emailKnown('lloyd@nowhe.re', this.callback);
     },
-    " as known when it is only staged": function (r) {
+    " as known when it is only staged": function (err, r) {
+      assert.isNull(err);
       assert.strictEqual(r, false);
     }
   }
@@ -121,8 +127,8 @@ suite.addBatch({
     topic: function() {
       db.gotVerificationSecret(secret, 'fakepasswordhash', this.callback);
     },
-    "gotVerificationSecret completes without error": function (r) {
-      assert.strictEqual(r, undefined);
+    "gotVerificationSecret completes without error": function (err, r) {
+      assert.isNull(err);
     }
   }
 });
@@ -132,7 +138,8 @@ suite.addBatch({
     topic: function() {
       db.isStaged('lloyd@nowhe.re', this.callback);
     },
-    "as staged immediately after its verified": function (r) {
+    "as staged immediately after its verified": function (err, r) {
+      assert.isNull(err);
       assert.strictEqual(r, false);
     }
   },
@@ -140,7 +147,8 @@ suite.addBatch({
     topic: function() {
       db.emailKnown('lloyd@nowhe.re', this.callback);
     },
-    "when it is": function (r) {
+    "when it is": function (err, r) {
+      assert.isNull(err);
       assert.strictEqual(r, true);
     }
   }
@@ -150,11 +158,12 @@ suite.addBatch({
   "checkAuth returns": {
     topic: function() {
       var cb = this.callback;
-      db.emailToUID('lloyd@nowhe.re', function(uid) {
+      db.emailToUID('lloyd@nowhe.re', function(err, uid) {
         db.checkAuth(uid, cb);
       });
     },
-    "the correct password": function(r) {
+    "the correct password": function(err, r) {
+      assert.isNull(err);
       assert.strictEqual(r, "fakepasswordhash");
     }
   }
@@ -165,14 +174,16 @@ suite.addBatch({
     topic: function() {
       db.emailToUID('lloyd@nowhe.re', this.callback);
     },
-    "returns a valid userid": function(r) {
+    "returns a valid userid": function(err, r) {
+      assert.isNull(err);
       assert.isNumber(r);
     },
     "returns a UID": {
-      topic: function(uid) {
+      topic: function(err, uid) {
         db.userOwnsEmail(uid, 'lloyd@nowhe.re', this.callback);
       },
-      "that owns the original email": function(r) {
+      "that owns the original email": function(err, r) {
+        assert.isNull(err);
         assert.ok(r);
       }
     }
@@ -180,38 +191,48 @@ suite.addBatch({
 });
 
 suite.addBatch({
-  "getting a UID, then": {
+  "getting a UID": {
     topic: function() {
       db.emailToUID('lloyd@nowhe.re', this.callback);
     },
-    "staging an email": {
-      topic: function(uid) {
+    "does not error": function(err, uid) {
+      assert.isNull(err);
+    },
+    "then staging an email": {
+      topic: function(err, uid) {
         db.stageEmail(uid, 'lloyd@somewhe.re', this.callback);
       },
-      "yields a valid secret": function(secret) {
+      "yields a valid secret": function(err, secret) {
+        assert.isNull(err);
         assert.isString(secret);
         assert.strictEqual(secret.length, 48);
       },
       "then": {
-        topic: function(secret) {
+        topic: function(err, secret) {
           var cb = this.callback;
-          db.isStaged('lloyd@somewhe.re', function(r) { cb(secret, r); });
+          db.isStaged('lloyd@somewhe.re', function(err, r) { cb(secret, r); });
         },
         "makes it visible via isStaged": function(sekret, r) { assert.isTrue(r); },
         "lets you verify it": {
           topic: function(secret, r) {
             db.gotVerificationSecret(secret, undefined, this.callback);
           },
-          "successfully": function(r) {
-            assert.isUndefined(r);
+          "successfully": function(err, r) {
+            assert.isNull(err);
           },
           "and knownEmail": {
             topic: function() { db.emailKnown('lloyd@somewhe.re', this.callback); },
-            "returns true": function(r) { assert.isTrue(r); }
+            "returns true": function(err, r) {
+              assert.isNull(err);
+              assert.isTrue(r);
+            }
           },
           "and isStaged": {
             topic: function() { db.isStaged('lloyd@somewhe.re', this.callback); },
-            "returns false": function(r) { assert.isFalse(r); }
+            "returns false": function(err, r) {
+              assert.isNull(err);
+              assert.isFalse(r);
+            }
           }
         }
       }
@@ -226,7 +247,8 @@ suite.addBatch({
       topic: function() {
         db.emailsBelongToSameAccount('lloyd@nowhe.re', 'lloyd@somewhe.re', this.callback);
       },
-      "when they do": function(r) {
+      "when they do": function(err, r) {
+        assert.isNull(err);
         assert.isTrue(r);
       }
     },
@@ -234,7 +256,8 @@ suite.addBatch({
       topic: function() {
         db.emailsBelongToSameAccount('lloyd@anywhe.re', 'lloyd@somewhe.re', this.callback);
       },
-      "when they don't": function(r) {
+      "when they don't": function(err, r) {
+        assert.isNull(err);
         assert.isFalse(r);
       }
     }
@@ -246,7 +269,8 @@ suite.addBatch({
     topic: function() {
       db.emailType('lloyd@anywhe.re', this.callback);
     },
-    "is null": function (r) {
+    "is null": function (err, r) {
+      assert.isNull(err);
       assert.isUndefined(r);
     }
   },
@@ -254,7 +278,8 @@ suite.addBatch({
     topic: function() {
       db.emailType('lloyd@somewhe.re', this.callback);
     },
-    "is 'secondary'": function (r) {
+    "is 'secondary'": function (err, r) {
+      assert.isNull(err);
       assert.strictEqual(r, 'secondary');
     }
   },
@@ -262,7 +287,8 @@ suite.addBatch({
     topic: function() {
       db.emailType('lloyd@nowhe.re', this.callback);
     },
-    "is 'secondary'": function (r) {
+    "is 'secondary'": function (err, r) {
+      assert.isNull(err);
       assert.strictEqual(r, 'secondary');
     }
   }
@@ -272,18 +298,20 @@ suite.addBatch({
   "removing an existing email": {
     topic: function() {
       var cb = this.callback;
-      db.emailToUID("lloyd@somewhe.re", function(uid) {
+      db.emailToUID("lloyd@somewhe.re", function(err, uid) {
         db.removeEmail(uid, "lloyd@nowhe.re", cb);
       });
     },
-    "returns no error": function(r) {
+    "returns no error": function(err, r) {
+      assert.isNull(err);
       assert.isUndefined(r);
     },
     "causes emailKnown": {
       topic: function() {
         db.emailKnown('lloyd@nowhe.re', this.callback);
       },
-      "to return false": function (r) {
+      "to return false": function (err, r) {
+        assert.isNull(err);
         assert.strictEqual(r, false);
       }
     }
@@ -295,14 +323,15 @@ suite.addBatch({
     topic: function() {
       db.createUserWithPrimaryEmail("lloyd@primary.domain", this.callback);
     },
-    "returns no error": function(r) {
-      assert.isUndefined(r);
+    "returns no error": function(err, r) {
+      assert.isNull(err);
     },
     "causes emailKnown": {
       topic: function() {
         db.emailKnown('lloyd@primary.domain', this.callback);
       },
-      "to return true": function (r) {
+      "to return true": function (err, r) {
+        assert.isNull(err);
         assert.strictEqual(r, true);
       }
     },
@@ -310,7 +339,8 @@ suite.addBatch({
       topic: function() {
         db.emailType('lloyd@primary.domain', this.callback);
       },
-      "to return 'primary'": function (r) {
+      "to return 'primary'": function (err, r) {
+        assert.isNull(err);
         assert.strictEqual(r, 'primary');
       }
     }
@@ -321,18 +351,19 @@ suite.addBatch({
   "adding a primary email to that account": {
     topic: function() {
       var cb = this.callback;
-      db.emailToUID('lloyd@primary.domain', function(uid) {
+      db.emailToUID('lloyd@primary.domain', function(err, uid) {
         db.addPrimaryEmailToAccount(uid, "lloyd2@primary.domain", cb);
       });
     },
-    "returns no error": function(r) {
-      assert.isUndefined(r);
+    "returns no error": function(err) {
+      assert.isNull(err);
     },
     "causes emailKnown": {
       topic: function() {
         db.emailKnown('lloyd2@primary.domain', this.callback);
       },
-      "to return true": function (r) {
+      "to return true": function (err, r) {
+        assert.isNull(err);
         assert.strictEqual(r, true);
       }
     },
@@ -340,7 +371,8 @@ suite.addBatch({
       topic: function() {
         db.emailType('lloyd@primary.domain', this.callback);
       },
-      "to return 'primary'": function (r) {
+      "to return 'primary'": function (err, r) {
+        assert.isNull(err);
         assert.strictEqual(r, 'primary');
       }
     }
@@ -348,18 +380,19 @@ suite.addBatch({
   "adding a primary email to an account with only secondaries": {
     topic: function() {
       var cb = this.callback;
-      db.emailToUID('lloyd@somewhe.re', function(uid) {
+      db.emailToUID('lloyd@somewhe.re', function(err, uid) {
         db.addPrimaryEmailToAccount(uid, "lloyd3@primary.domain", cb);
       });
     },
-    "returns no error": function(r) {
-      assert.isUndefined(r);
+    "returns no error": function(err) {
+      assert.isNull(err);
     },
     "causes emailKnown": {
       topic: function() {
         db.emailKnown('lloyd3@primary.domain', this.callback);
       },
-      "to return true": function (r) {
+      "to return true": function (err, r) {
+        assert.isNull(err);
         assert.strictEqual(r, true);
       }
     },
@@ -367,7 +400,8 @@ suite.addBatch({
       topic: function() {
         db.emailType('lloyd3@primary.domain', this.callback);
       },
-      "to return 'primary'": function (r) {
+      "to return 'primary'": function (err, r) {
+        assert.isNull(err);
         assert.strictEqual(r, 'primary');
       }
     }
@@ -378,18 +412,19 @@ suite.addBatch({
   "adding a registered primary email to an account": {
     topic: function() {
       var cb = this.callback;
-      db.emailToUID('lloyd@primary.domain', function(uid) {
+      db.emailToUID('lloyd@primary.domain', function(err, uid) {
         db.addPrimaryEmailToAccount(uid, "lloyd3@primary.domain", cb);
       });
     },
-    "returns no error": function(r) {
-      assert.isUndefined(r);
+    "returns no error": function(err) {
+      assert.isNull(err);
     },
     "and emailKnown": {
       topic: function() {
         db.emailKnown('lloyd3@primary.domain', this.callback);
       },
-      "still returns true": function (r) {
+      "still returns true": function (err, r) {
+        assert.isNull(err);
         assert.strictEqual(r, true);
       }
     },
@@ -397,7 +432,8 @@ suite.addBatch({
       topic: function() {
         db.emailType('lloyd@primary.domain', this.callback);
       },
-      "still returns 'primary'": function (r) {
+      "still returns 'primary'": function (err, r) {
+        assert.isNull(err);
         assert.strictEqual(r, 'primary');
       }
     },
@@ -405,7 +441,8 @@ suite.addBatch({
       topic: function() {
         db.emailsBelongToSameAccount('lloyd3@primary.domain', 'lloyd@somewhe.re', this.callback);
       },
-      "from original account": function(r) {
+      "from original account": function(err, r) {
+        assert.isNull(err);
         assert.isFalse(r);
       }
     },
@@ -413,7 +450,8 @@ suite.addBatch({
       topic: function() {
         db.emailsBelongToSameAccount('lloyd3@primary.domain', 'lloyd@primary.domain', this.callback);
       },
-      "to new account": function(r) {
+      "to new account": function(err, r) {
+        assert.isNull(err);
         assert.isTrue(r);
       }
     }
@@ -424,18 +462,19 @@ suite.addBatch({
   "canceling an account": {
     topic: function() {
       var cb = this.callback;
-      db.emailToUID("lloyd@somewhe.re", function(uid) {
+      db.emailToUID("lloyd@somewhe.re", function(err, uid) {
         db.cancelAccount(uid, cb);
       });
     },
-    "returns no error": function(r) {
-      assert.isUndefined(r);
+    "returns no error": function(err) {
+      assert.isNull(err);
     },
     "causes emailKnown": {
       topic: function() {
         db.emailKnown('lloyd@somewhe.re', this.callback);
       },
-      "to return false": function (r) {
+      "to return false": function (err, r) {
+        assert.isNull(err);
         assert.strictEqual(r, false);
       }
     }
@@ -448,21 +487,21 @@ suite.addBatch({
       db.close(this.callback);
     },
     "should work": function(err) {
-      assert.isUndefined(err);
+      assert.isNull(err);
     },
     "re-opening the database": {
       topic: function() {
         db.open(dbCfg, this.callback);
       },
-      "works": function(r) {
-        assert.isUndefined(r);
+      "works": function(err) {
+        assert.isNull(err);
       },
       "and then purging": {
         topic: function() {
           db.closeAndRemove(this.callback);
         },
         "works": function(r) {
-          assert.isUndefined(r);
+          assert.isNull(r);
         }
       }
     }
diff --git a/tests/lib/start-stop.js b/tests/lib/start-stop.js
index d7485e469285845f34916b3f1d7a34675a3f0849..895ae04f4ec1c039b32a590e8d0351084cf5acfd 100644
--- a/tests/lib/start-stop.js
+++ b/tests/lib/start-stop.js
@@ -103,7 +103,7 @@ exports.addStartupBatches = function(suite) {
         db.open(cfg, this.callback);
       },
       "should work fine": function(r) {
-        assert.isUndefined(r);
+        assert.isNull(r);
       }
     }
   });
@@ -184,7 +184,7 @@ exports.addShutdownBatches = function(suite) {
         db.closeAndRemove(this.callback);
       },
       "should work": function(err) {
-        assert.isUndefined(err);
+        assert.isNull(err);
       }
     }
   });
diff --git a/tests/password-bcrypt-update-test.js b/tests/password-bcrypt-update-test.js
index db6a994957ec49641568b68d3ccc43655e9f2f75..c403d6ef715d1250b6671128abd9a64b1b7af923 100755
--- a/tests/password-bcrypt-update-test.js
+++ b/tests/password-bcrypt-update-test.js
@@ -87,11 +87,12 @@ suite.addBatch({
   "the password": {
     topic: function() {
       var cb = this.callback;
-      db.emailToUID(TEST_EMAIL, function(uid) {
+      db.emailToUID(TEST_EMAIL, function(err, uid) {
         db.checkAuth(uid, cb);
       });
     },
-    "is bcrypted with the expected number of rounds": function(r) {
+    "is bcrypted with the expected number of rounds": function(err, r) {
+      assert.isNull(err);
       assert.equal(typeof r, 'string');
       assert.equal(config.get('bcrypt_work_factor'), bcrypt.get_rounds(r));
     }
@@ -134,11 +135,12 @@ suite.addBatch({
     "if we recheck the auth hash": {
       topic: function() {
         var cb = this.callback;
-        db.emailToUID(TEST_EMAIL, function(uid) {
+        db.emailToUID(TEST_EMAIL, function(err, uid) {
           db.checkAuth(uid, cb);
         });
       },
-      "its bcrypted with 8 rounds": function(r) {
+      "its bcrypted with 8 rounds": function(err, r) {
+        assert.isNull(err);
         assert.equal(typeof r, 'string');
         assert.equal(8, bcrypt.get_rounds(r));
       }
diff --git a/tests/stalled-mysql-test.js b/tests/stalled-mysql-test.js
new file mode 100755
index 0000000000000000000000000000000000000000..e0f4f60186c921da46c31aec8335f3a09216d47d
--- /dev/null
+++ b/tests/stalled-mysql-test.js
@@ -0,0 +1,371 @@
+#!/usr/bin/env node
+
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+require('./lib/test_env.js');
+
+if (process.env['NODE_ENV'] != 'test_mysql') process.exit(0);
+
+const assert = require('assert'),
+vows = require('vows'),
+start_stop = require('./lib/start-stop.js'),
+wsapi = require('./lib/wsapi.js'),
+temp = require('temp'),
+fs = require('fs'),
+jwk = require('jwcrypto/jwk.js'),
+jwt = require('jwcrypto/jwt.js'),
+vep = require('jwcrypto/vep.js'),
+jwcert = require('jwcrypto/jwcert.js'),
+path = require('path');
+
+var suite = vows.describe('forgotten-email');
+
+// disable vows (often flakey?) async error behavior
+suite.options.error = false;
+
+// let's reduce the amount of time allowed for queries, so that
+// we get a faster failure and tests run quicker
+process.env['MAX_QUERY_TIME_MS'] = 250;
+
+// and let's instruct children to pretend as if the driver is
+// stalled if a file exists
+var stallFile = temp.path({suffix: '.stall'});
+process.env['STALL_MYSQL_WHEN_PRESENT'] = stallFile;
+
+start_stop.addStartupBatches(suite);
+
+// ever time a new token is sent out, let's update the global
+// var 'token'
+var token = undefined;
+
+function addStallDriverBatch(stall) {
+  suite.addBatch({
+    "changing driver state": {
+      topic: function() {
+        if (stall) fs.writeFileSync(stallFile, "");
+        else fs.unlinkSync(stallFile);
+        setTimeout(this.callback, 300);
+      },
+      "completes": function(err, r) { }
+    }
+  });
+}
+
+// first stall mysql
+addStallDriverBatch(true);
+
+// call session context once to populate CSRF stuff in the
+// wsapi client lib
+suite.addBatch({
+  "get context": {
+    topic: wsapi.get('/wsapi/session_context'),
+    "works" : function(err, r) {
+      assert.isNull(err);
+    }
+  }
+});
+
+// now try all apis that can be excercised without further setup
+suite.addBatch({
+  "address_info": {
+    topic: wsapi.get('/wsapi/address_info', {
+      email: 'test@example.domain'
+    }),
+    "works": function(err, r) {
+      // address info with a primary address doesn't need db access.
+      assert.strictEqual(r.code, 200);
+    }
+  },
+  "address_info": {
+    topic: wsapi.get('/wsapi/address_info', {
+      email: 'test@hilaiel.com'
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "have_email": {
+    topic: wsapi.get('/wsapi/have_email', {
+      email: 'test@example.com'
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "authenticate_user": {
+    topic: wsapi.post('/wsapi/authenticate_user', {
+      email: 'test@example.com',
+      pass: 'oogabooga'
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "complete_email_addition": {
+    topic: wsapi.post('/wsapi/complete_email_addition', {
+      token: 'bogus'
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "complete_user_creation": {
+    topic: wsapi.post('/wsapi/complete_user_creation', {
+      token: 'bogus',
+      pass: 'fakefake'
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "email_for_token": {
+    topic: wsapi.get('/wsapi/email_for_token', {
+      token: 'bogus'
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "stage_user": {
+    topic: wsapi.post('/wsapi/stage_user', {
+      email: 'bogus@bogus.edu',
+      site: 'whatev.er'
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  }
+});
+
+// now unstall the driver, we'll create an account and sign in in
+// order to test the behavior of the remaining APIs when the database
+// is stalled
+addStallDriverBatch(false);
+
+var token = undefined;
+
+suite.addBatch({
+  "account staging": {
+    topic: wsapi.post('/wsapi/stage_user', {
+      email: "stalltest@whatev.er",
+      site: 'fakesite.com'
+    }),
+    "works":     function(err, r) {
+      assert.equal(r.code, 200);
+    }
+  }
+});
+
+suite.addBatch({
+  "a token": {
+    topic: function() {
+      start_stop.waitForToken(this.callback);
+    },
+    "is obtained": function (t) {
+      assert.strictEqual(typeof t, 'string');
+    },
+    "setting password": {
+      topic: function(token) {
+        wsapi.post('/wsapi/complete_user_creation', {
+          token: token,
+          pass: "somepass"
+        }).call(this);
+      },
+      "works just fine": function(err, r) {
+        assert.equal(r.code, 200);
+      }
+    }
+  }
+});
+
+// re-stall mysql
+addStallDriverBatch(true);
+
+// test remaining wsapis
+
+suite.addBatch({
+  "account_cancel": {
+    topic: wsapi.post('/wsapi/account_cancel', { }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "cert_key": {
+    topic: wsapi.post('/wsapi/cert_key', {
+      email: "test@whatev.er",
+      pubkey: "bogus"
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "email_addition_status": {
+    topic: wsapi.get('/wsapi/email_addition_status', {
+      email: "test@whatev.er"
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "list_emails": {
+    topic: wsapi.get('/wsapi/list_emails', {}),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "remove_email": {
+    topic: wsapi.post('/wsapi/remove_email', {
+      email: "test@whatev.er"
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "session_context": {
+    topic: wsapi.get('/wsapi/session_context', { }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "stage_email": {
+    topic: wsapi.post('/wsapi/stage_email', {
+      email: "test2@whatev.er",
+      site: "foo.com"
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "update_password": {
+    topic: wsapi.post('/wsapi/update_password', {
+      oldpass: "oldpassword",
+      newpass: "newpassword"
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "user_creation_status": {
+    topic: wsapi.get('/wsapi/user_creation_status', {
+      email: "test3@whatev.er"
+    }),
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  }
+});
+
+// now let's test apis that require an assertion, and only after verifying
+// that, hit the database
+const TEST_DOMAIN = 'example.domain',
+      TEST_EMAIL = 'testuser@' + TEST_DOMAIN,
+      TEST_ORIGIN = 'http://127.0.0.1:10002',
+      TEST_FIRST_ACCT = 'testuser@fake.domain';
+
+var g_keypair, g_cert, g_assertion;
+
+suite.addBatch({
+  "generating a keypair": {
+    topic: function() {
+      return jwk.KeyPair.generate("DS", 256)
+    },
+    "succeeds": function(r, err) {
+      assert.isObject(r);
+      assert.isObject(r.publicKey);
+      assert.isObject(r.secretKey);
+      g_keypair = r;
+    }
+  }
+});
+
+var g_privKey = jwk.SecretKey.fromSimpleObject(
+  JSON.parse(require('fs').readFileSync(
+    path.join(__dirname, '..', 'example', 'primary', 'sample.privatekey'))));
+
+
+suite.addBatch({
+  "generting a certificate": {
+    topic: function() {
+      var domain = process.env['SHIMMED_DOMAIN'];
+
+      var expiration = new Date();
+      expiration.setTime(new Date().valueOf() + 60 * 60 * 1000);
+      g_cert = new jwcert.JWCert(TEST_DOMAIN, expiration, new Date(),
+                                 g_keypair.publicKey, {email: TEST_EMAIL}).sign(g_privKey);
+      return g_cert;
+    },
+    "works swimmingly": function(cert, err) {
+      assert.isString(cert);
+      assert.lengthOf(cert.split('.'), 3);
+    }
+  }
+});
+
+suite.addBatch({
+  "generating an assertion": {
+    topic: function() {
+      var expirationDate = new Date(new Date().getTime() + (2 * 60 * 1000));
+      var tok = new jwt.JWT(null, expirationDate, TEST_ORIGIN);
+      return vep.bundleCertsAndAssertion([g_cert], tok.sign(g_keypair.secretKey));
+    },
+    "succeeds": function(r, err) {
+      assert.isString(r);
+      g_assertion = r;
+    }
+  }
+});
+
+// finally!  we have our assertion in g_assertion
+suite.addBatch({
+  "add_email_with_assertion": {
+    topic: function() {
+      wsapi.post('/wsapi/add_email_with_assertion', {
+        assertion: g_assertion
+      }).call(this);
+    },
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "auth_with_assertion": {
+    topic: function() {
+      wsapi.post('/wsapi/auth_with_assertion', {
+        assertion: g_assertion
+      }).call(this);
+    },
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  },
+  "create_account_with_assertion": {
+    topic: function() {
+      wsapi.post('/wsapi/create_account_with_assertion', {
+        assertion: g_assertion
+      }).call(this);
+    },
+    "fails with 503": function(err, r) {
+      assert.strictEqual(r.code, 503);
+    }
+  }
+});
+
+// logout doesn't need database, it should still succeed
+suite.addBatch({
+  "logout": {
+    topic: wsapi.post('/wsapi/logout', { }),
+    "succeeds": function(err, r) {
+      assert.strictEqual(r.code, 200);
+    }
+  }
+});
+
+// finally, unblock mysql so we can shut down
+addStallDriverBatch(false);
+
+start_stop.addShutdownBatches(suite);
+
+// run or export the suite.
+if (process.argv[1] === __filename) suite.run();
+else suite.export(module);