improve documentation of mysql wrapper and associated tests, remove debug...

improve documentation of mysql wrapper and associated tests, remove debug console output, fail slow queries rather than retrying, and add a statsd counter for failed_query

lib/db/mysql_wrapper.js

@@ -77,7 +77,10 @@ exports.createClient = function(options) {
           slowQueryTimer = null;
         } else {
           logger.warn("Query taking more than " + config.get('database.max_query_time_ms') + "ms!  reconnecting to mysql");
-          queryQueue.unshift(work);
+          // we'll fail the long running query, because we cannot
+          // meaningfully know whether or not it completed in the case where
+          // the driver is unresponsive.
+          invokeCallback(work.cb, "database connection unavailable");
           self._resetConnection();
           slowQueryTimer = null;
           self._runNextQuery();
@@ -93,9 +96,13 @@ exports.createClient = function(options) {
         slowQueryTimer = null;
         consecutiveFailures = 0;
 
+        // report query time for all queries via statsd
         var reqTime = new Date - work.startTime;
         statsd.timing('query_time', reqTime);
 
+        // report failed queries via statsd
+        if (err) statsd.increment('failed_query'); 
+
         invokeCallback(work.cb, err, r);
         self._runNextQuery();
       });

lib/wsapi/add_email_with_assertion.js

@@ -24,7 +24,6 @@ exports.i18n = false;
 exports.process = function(req, res) {
   // first let's verify that the assertion is valid
   primary.verifyAssertion(req.body.assertion, function(err, email) {
-    console.log("MOTHERFUCKER", err);
     if (err) {
       return res.json({
         success: false,

tests/stalled-mysql-test.js

@@ -46,6 +46,14 @@ function addStallDriverBatch(stall) {
       topic: function() {
         if (stall) fs.writeFileSync(stallFile, "");
         else fs.unlinkSync(stallFile);
+
+        // After changing the file which indicates to child
+        // processes whether the driver should simulate a stalled
+        // state or not, we need to wait for them to detect the
+        // change.  because we use `fs.watchFile()` on a short poll,
+        // this should be nearly instantaneous.  300ms is a magic number
+        // which is hoped to allow plenty of time even on a loaded
+        // machine
         setTimeout(this.callback, 300);
       },
       "completes": function(err, r) { }