load_gen

#!/usr/bin/env node

/* ***** BEGIN LICENSE BLOCK *****
 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Original Code is Mozilla BrowserID.
 *
 * The Initial Developer of the Original Code is Mozilla.
 * Portions created by the Initial Developer are Copyright (C) 2011
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s):
 *   Lloyd Hilaiel <lloyd@hilaiel.com>
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 2 or later (the "GPL"), or
 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 * in which case the provisions of the GPL or the LGPL are applicable instead
 * of those above. If you wish to allow use of your version of this file only
 * under the terms of either the GPL or the LGPL, and not to allow others to
 * use your version of this file under the terms of the MPL, indicate your
 * decision by deleting the provisions above and replace them with the notice
 * and other provisions required by the GPL or the LGPL. If you do not delete
 * the provisions above, a recipient may use your version of this file under
 * the terms of any one of the MPL, the GPL or the LGPL.
 *
 * ***** END LICENSE BLOCK ***** */

/* This file is the main run file for the browserid load generation
 * tool, which is capable of analysing the maximum active users that
 * a browserid deployment can support */

const winston = require('winston');

// option processing with optimist
var argv = require('optimist')
.usage('Apply load to a BrowserID server.\nUsage: $0', [ "foo" ])
.alias('a', 'activities')
.describe('a', 'only run a subset of activities, specified as a CSV list')
.alias('h', 'help')
.describe('h', 'display this usage message')
.alias('l', 'list')
.describe('l', 'list available activities and exit')
.alias('m', 'max')
.describe('m', 'maximum active users to simulate (0 == infinite)')
.default('m', 10000)
.alias('o', 'omit-static')
.describe('o', 'when enabled, only dynamic WSAPI calls will be simulated, not static resource requests')
.default('o', false)
.alias('s', 'server')
.string('s')
.describe('s', 'base URL to browserid server')
.check(function(argv) {
  return (argv.h || typeof argv.s === 'string' || argv.l) != undefined;
})
.alias('v', 'verifier')
.describe('v', 'base URL to verifier service (default is browserid server + \'/verify\')')
.alias('u', 'user-range')
.describe('u', 'rather than creating users, assume a range of users exist #@loadtest.domain (with password "THE PASSWORD")');

var args = argv.argv;

// global configuration
const configuration = {
  verifier: args.v ? args.v : args.s + "/verify",
  browserid: args.s
};

// last time we updated stats and added work if necc.
var lastPoll = new Date();

// average active users simulated over the last second, 5s, and 60s
var averages = [
  0.0,
  0.0,
  0.0
];

// activities complete since the last poll
var completed = {
};

// how many activies does an active user undertake per second
const activitiesPerUserPerSecond = (40.0 / ( 24 * 60 * 60 ));

// activities
var activity = {
  "signup": {
    // a %20 montly growth rate means there's a 20% probability of
    // the monthly activity generated by an active user being a
    // new user signup
    probability: (1.0 / (40 * 28 * .2))
  },
  "reset_pass": {
    // users forget their password once every 4 weeks
    probability: (1.0 / (40 * 28.0))
  },
  "add_email": {
    // users add a new email address once every 2 weeks
    probability: (1.0 / (40 * 14.0))
  },
  "reauth": {
    // users must re-authenticate to browser id once a week
    // (once every two weeks per device)
    probability: (1.0 / (40 * 7.0))
  },
  "signin": {
    // users sign in using existing authentication material
    // 8 times a day (once every six hours per device)
    probability: (8 / 40.0)
  },
  "include_only": {
    // most of the time, users are already authenticated to their
    // RPs, so the hit on our servers is simply resource (include.js)
    // inclusion.  The strict probability is 100% - sum of above
    // probabilities.  We round to 31 / 40.
    probability: (31 / 40.0)
  },
  "change_pass": {
    // users change their passwords once every two months
    probability: (1.0 / (40 * 56))
  }
};

if (args.l) {
  console.log("available activities:", Object.keys(activity).join(", "));
  process.exit(0);
}

if (args.h) {
  argv.showHelp();
  process.exit(1);
}

var activitiesToRun = Object.keys(activity);

// handle modification of activities to run (-o or -a)
if (args.a) {
  if (typeof args.a !== 'string') {
    process.stdout.write("invalid argument: " + args.a.toString() + "\n\n");
    argv.showHelp();
    process.exit(1);
  }
  activitiesToRun = args.a.split(',');
  activitiesToRun.forEach(function(act) {
    if (!activity.hasOwnProperty(act)) {
      process.stdout.write("invalid activity: " + act + "\n\n");
      process.exit(1);
    }
  });
} else if (args.o) {
  activitiesToRun.splice(activitiesToRun.indexOf('include_only'), 1);
}

// outstanding incomplete activites
var outstanding = { };

Object.keys(activity).forEach(function(act) {
  outstanding[act] = 0;
});

function numOutstanding() {
  var n = 0;
  Object.keys(outstanding).forEach(function(act) {
    n += outstanding[act];
  });
  return n;
}

// now attach "start functions" to the activity map by including
// the implementation of each activity
Object.keys(activity).forEach(function(k) {
  activity[k].startFunc = require("../lib/load_gen/activities/" + k).startFunc;
});

// probs is a 2d array mapping normalized probabilities from 0-1 to
// activities, used when determining what activity to perform next
var probs = [];
Object.keys(activity).forEach(function(k) {
  var sum = 0;
  if (probs.length) sum = probs[probs.length - 1][0];
  sum += activity[k].probability;
  probs.push([sum, k]);
});

// and normalize probs into 0..1
(function() {
  var max = probs[probs.length - 1][0];
  for (var i = 0; i < probs.length; i++) {
    probs[i][0] /= max;
  }
})();

// a global count of how many poll iterations have been completed
var iterations = 0;

// output a textual summary of how many activites per second are
// associated with the given number of active users
function outputActiveUserSummary(activeUsers) {
  console.log("with", activeUsers, "active users there will be:");
  for (var i = 0; i < probs.length; i++) {
    var p = probs[i][0];
    if (i !== 0) p -= probs[i-1][0];
    var n = p * activeUsers * activitiesPerUserPerSecond;
    console.log(" ", n.toFixed(2), probs[i][1], "activites per second");
  }
}

function poll() {
  function startNewActivity() {
    // what type of activity is this?
    var n = Math.random();
    var act = undefined;
    for (var i = 0; i < probs.length; i++) {
      if (n <= probs[i][0]) {
        act = probs[i][1];
        break;
      }
    }
    // start the activity! (if it is enabled)
    if (activitiesToRun.indexOf(act) !== -1) {
      outstanding[act]++;
      activity[act].startFunc(configuration, function(err) {
        outstanding[act]--;
        if (undefined === completed[act]) completed[act] = [ 0, 0, 0 ];
        if (err) {
          if (typeof err != 'string') err = err.toString();
          if (err.indexOf('server is too busy') != -1) {
            completed[act][2]++;
          } else {
            completed[act][1]++;
            winston.error('('+act+') ' + err);
          }
        } else {
          completed[act][0]++;
        }
      });
    } else {
      if (undefined === completed[act]) completed[act] = [ 0, 0, 0 ];
      completed[act][0]++;
    }
  }

  var numErrors = 0;
  var num503s = 0;
  var numStarted = 0;

  function updateAverages(elapsed) {
    if (!iterations) return;

    var numActCompleted = 0;
    Object.keys(completed).forEach(function(k) {
      numActCompleted += completed[k][0];
      numErrors += completed[k][1];
      num503s += completed[k][2];
    });
    completed = { };
    var avgUsersThisPeriod = (numActCompleted / activitiesPerUserPerSecond) * (elapsed / 1000);

    // the 1s average is a goldfish.
    averages[0] = avgUsersThisPeriod;

    // for 5s and 60s averages, a little special logic to handle cases
    // where we don't have enough history to dampen based on past performance
    var i = 5 > iterations ? iterations * 1.0 : 5.0;
    averages[1] = ((i-1) * averages[1] + avgUsersThisPeriod) / i;
    var i = 60 > iterations ? iterations * 1.0 : 60.0;
    averages[2] = ((i-1) * averages[2] + avgUsersThisPeriod) / i;
  }

  function outputAverages() {
    var actSumString = numOutstanding() + " R, " + numStarted + " S";
    var actNums = [];
    Object.keys(outstanding).forEach(function(act) {
      actNums.push(outstanding[act] + act.substr(0,1) + act.substr(-1,1));
    });
    actSumString += " (" + actNums.join(' ') + ")";

    console.log("\t", averages[0].toFixed(2),
                "\t", averages[1].toFixed(2),
                "\t", averages[2].toFixed(2),
                "\t", actSumString,
                "\t", numErrors ? "(" + numErrors + " ERRORS!)" : "",
                "\t", num503s ? " (" + num503s + " 503s)" : "");
  }

  // ** how much time has elapsed since the last poll?
  var elapsed;
  {
    var now = new Date();
    elapsed = now - lastPoll;
    lastPoll = now;
  }

  // ** update running averages **
  updateAverages(elapsed);

  // ** determine how many activities to start **

  // how many active users would we like to simulate
  var targetActive = args.m;

  // if we're not throttled, then we'll trying 150% as many as
  // we're simulating right now.  If we're not simulating at least
  // 10000 active users, that shall be our lower bound
  if (!targetActive) {
    if (averages[0] > 10000) targetActive = averages[0] * 1.5;
    else targetActive = 10000;
  }

  // now how many new activities do we want to start?
  var newAct = activitiesPerUserPerSecond * targetActive;

  // scale based on how much time has elapsed since the last poll
  // on every iteration except the first
  if (iterations) newAct *= (elapsed / 1000);

  // probabilistic rounding
  {
    var add = (newAct % 1.0) < Math.random() ? 0 : 1;
    newAct = Math.floor(newAct) + add;
  }

  // ** start activities **

  // start the new activites until they're all started, or until we've
  // got twice as many outstanding as would be required by the target we
  // want to hit (which means the server can't keep up).
  while (newAct >= 1.0 && numOutstanding() < (activitiesPerUserPerSecond * targetActive * 2)) {
    numStarted++;
    startNewActivity();
    newAct--;
  }

  // ** schedule another wake up
  var wakeUpIn = 1000 - (new Date() - lastPoll);
  setTimeout(poll, wakeUpIn);

  // display averages
  outputAverages();

  iterations++;
}

// always start out by creating a bunch of users
var NUM_INITIAL_USERS = 100;

// if an explicit target was specified, let's output what that means
// in understandable terms
if (args.m) outputActiveUserSummary(args.m);

const userdb = require("../lib/load_gen/user_db.js");

if (args.u) {
  // parse args.u
  var start, end;
  try {
    var r = args.u.split('/');
    if (r.length != 2) throw "expected format ##/##";
    start = parseInt(r[0], 10);
    end = parseInt(r[1], 10);
    if (start >= end) throw "first number must be smaller than the second";
  } catch(e) {
    console.log("your -u argument is poorly formated: " + e.toString());
    process.exit(1);
  }

  // now create all them users!
  console.log("Runing with", (end - start + 1), "pre-created users (XXX@loadtest.domain)");
  for (var i = start; i < end; i++) {
    userdb.addNewUser(userdb.getNewUser(i + "@loadtest.domain", "THE PASSWORD"));
  }
  console.log("users created!  applying load...");
  poll();
} else {
  console.log("To start, let's create " + NUM_INITIAL_USERS + " users via the API.  One moment please...");

  var createUser = require("../lib/load_gen/activities/signup.js").startFunc;
  var created = 0;
  for (var i = 0; i < NUM_INITIAL_USERS; i++) {
    createUser(configuration, function(err) {
      if (err) {
        console.log("failed to create initial users! tragedy!  run away!:", err);
        process.exit(1);
      }
      process.stdout.write(".");
      if (++created == NUM_INITIAL_USERS) {
        process.stdout.write("\n\n");
        console.log("Average active users simulated over the last 1s/5s/60s:");
        poll();
      }
    });
  }
}