load_gen

#!/usr/bin/env node

/* ***** BEGIN LICENSE BLOCK *****
 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Original Code is Mozilla BrowserID.
 *
 * The Initial Developer of the Original Code is Mozilla.
 * Portions created by the Initial Developer are Copyright (C) 2011
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s):
 *   Lloyd Hilaiel <lloyd@hilaiel.com>
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 2 or later (the "GPL"), or
 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 * in which case the provisions of the GPL or the LGPL are applicable instead
 * of those above. If you wish to allow use of your version of this file only
 * under the terms of either the GPL or the LGPL, and not to allow others to
 * use your version of this file under the terms of the MPL, indicate your
 * decision by deleting the provisions above and replace them with the notice
 * and other provisions required by the GPL or the LGPL. If you do not delete
 * the provisions above, a recipient may use your version of this file under
 * the terms of any one of the MPL, the GPL or the LGPL.
 *
 * ***** END LICENSE BLOCK ***** */

/* This file is the main run file for the browserid load generation
 * tool, which is capable of analysing the maximum active users that
 * a browserid deployment can support */

const winston = require('winston');

// option processing with optimist
var argv = require('optimist')
.usage('Apply load to a BrowserID server.\nUsage: $0', [ "foo" ])
.alias('a', 'activities')
.describe('a', 'only run a subset of activities, specified as a CSV list')
.alias('h', 'help')
.describe('h', 'display this usage message')
.alias('l', 'list')
.describe('l', 'list available activities and exit')
.alias('m', 'max')
.describe('m', 'maximum active users to simulate (0 == infinite)')
.default('m', 10000)
.alias('o', 'omit-static')
.describe('o', 'when enabled, only dynamic WSAPI calls will be simulated, not static resource requests')
.default('o', false)
.alias('s', 'server')
.describe('s', 'base URL to browserid server')
.demand('s')
.alias('v', 'verifier')
.describe('v', 'base URL to verifier service (default is browserid server + \'/verify\')')
.alias('u', 'user-range')
.describe('u', 'rather than creating users, assume a range of users exist #@loadtest.domain (with password "THE PASSWORD")');

var args = argv.argv;

if (args.h) {
  argv.showHelp();
  process.exit(1);
}

// global configuration
const configuration = {
  verifier: args.v ? args.v : args.s + "/verify",
  browserid: args.s
};

// last time we updated stats and added work if necc.
var lastPoll = new Date();

// average active users simulated over the last second, 5s, and 60s
var averages = [
  0.0,
  0.0,
  0.0
];

// activities complete since the last poll
var completed = {
};

// how many activies does an active user undertake per second
const activitiesPerUserPerSecond = (40.0 / ( 24 * 60 * 60 ));

// activities
var activity = {
  "signup": {
    // a %20 montly growth rate means there's a 20% probability of
    // the monthly activity generated by an active user being a
    // new user signup
    probability: (1.0 / (40 * 28 * .2))
  },
  "reset_pass": {
    // users forget their password once every 4 weeks
    probability: (1.0 / (40 * 28.0))
  },
  "add_email": {
    // users add a new email address once every 2 weeks
    probability: (1.0 / (40 * 14.0))
  },
  "reauth": {
    // users must re-authenticate to browser id once a week
    // (once every two weeks per device)
    probability: (1.0 / (40 * 7.0))
  },
  "signin": {
    // users sign in using existing authentication material
    // 8 times a day (once every six hours per device)
    probability: (8 / 40.0)
  },
  "include_only": {
    // most of the time, users are already authenticated to their
    // RPs, so the hit on our servers is simply resource (include.js)
    // inclusion.  The strict probability is 100% - sum of above
    // probabilities.  We round to 31 / 40.
    probability: (31 / 40.0)
  },
  "change_pass": {
    // users change their passwords once every two months
    probability: (1.0 / (40 * 56))
  }
};

if (args.l) {
  console.log("available activities:", Object.keys(activity).join(", "));
  process.exit(0);
}

var activitiesToRun = Object.keys(activity);

// handle modification of activities to run (-o or -a)
if (args.a) {
  if (typeof args.a !== 'string') {
    process.stdout.write("invalid argument: " + args.a.toString() + "\n\n");
    argv.showHelp();
    process.exit(1);
  }
  activitiesToRun = args.a.split(',');
  activitiesToRun.forEach(function(act) {
    if (!activity.hasOwnProperty(act)) {
      process.stdout.write("invalid activity: " + act + "\n\n");
      process.exit(1);
    }
  });
} else if (args.o) {
  activitiesToRun.splice(activitiesToRun.indexOf('include_only'), 1);
}

// outstanding incomplete activites
var outstanding = { };

Object.keys(activity).forEach(function(act) {
  outstanding[act] = 0;
});

function numOutstanding() {
  var n = 0;
  Object.keys(outstanding).forEach(function(act) {
    n += outstanding[act];
  });
  return n;
}

// now attach "start functions" to the activity map by including
// the implementation of each activity
Object.keys(activity).forEach(function(k) {
  activity[k].startFunc = require("../lib/load_gen/activities/" + k).startFunc;
});

// probs is a 2d array mapping normalized probabilities from 0-1 to
// activities, used when determining what activity to perform next
var probs = [];
Object.keys(activity).forEach(function(k) {
  var sum = 0;
  if (probs.length) sum = probs[probs.length - 1][0];
  sum += activity[k].probability;
  probs.push([sum, k]);
});

// and normalize probs into 0..1
(function() {
  var max = probs[probs.length - 1][0];
  for (var i = 0; i < probs.length; i++) {
    probs[i][0] /= max;
  }
})();

// a global count of how many poll iterations have been completed
var iterations = 0;

// output a textual summary of how many activites per second are
// associated with the given number of active users
function outputActiveUserSummary(activeUsers) {
  console.log("with", activeUsers, "active users there will be:");
  for (var i = 0; i < probs.length; i++) {
    var p = probs[i][0];
    if (i !== 0) p -= probs[i-1][0];
    var n = p * activeUsers * activitiesPerUserPerSecond;
    console.log(" ", n.toFixed(2), probs[i][1], "activites per second");
  }
}

function poll() {
  function startNewActivity() {
    // what type of activity is this?
    var n = Math.random();
    var act = undefined;
    for (var i = 0; i < probs.length; i++) {
      if (n <= probs[i][0]) {
        act = probs[i][1];
        break;
      }
    }
    // start the activity! (if it is enabled)
    if (activitiesToRun.indexOf(act) !== -1) {
      outstanding[act]++;
      activity[act].startFunc(configuration, function(err) {
        if (err) winston.error(err);
        outstanding[act]--;
        if (undefined === completed[act]) completed[act] = [ 0, 0 ];
        completed[act][err ? 1 : 0]++;
      });
    } else {
      if (undefined === completed[act]) completed[act] = [ 0, 0 ];
      completed[act][0]++;
    }
  }

  var numErrors = 0;
  var numStarted = 0;

  function updateAverages(elapsed) {
    if (!iterations) return;

    var numActCompleted = 0;
    Object.keys(completed).forEach(function(k) {
      numActCompleted += completed[k][0];
      numErrors += completed[k][1];
    });
    completed = { };
    var avgUsersThisPeriod = (numActCompleted / activitiesPerUserPerSecond) * (elapsed / 1000);

    // the 1s average is a goldfish.
    averages[0] = avgUsersThisPeriod;

    // for 5s and 60s averages, a little special logic to handle cases
    // where we don't have enough history to dampen based on past performance
    var i = 5 > iterations ? iterations * 1.0 : 5.0;
    averages[1] = ((i-1) * averages[1] + avgUsersThisPeriod) / i;
    var i = 60 > iterations ? iterations * 1.0 : 60.0;
    averages[2] = ((i-1) * averages[2] + avgUsersThisPeriod) / i;
  }

  function outputAverages() {
    var actSumString = numOutstanding() + " R, " + numStarted + " S";
    var actNums = [];
    Object.keys(outstanding).forEach(function(act) {
      actNums.push(outstanding[act] + act.substr(0,1) + act.substr(-1,1));
    });
    actSumString += " (" + actNums.join(' ') + ")";

    console.log("\t", averages[0].toFixed(2),
                "\t", averages[1].toFixed(2),
                "\t", averages[2].toFixed(2),
                "\t", actSumString,
                "\t", numErrors ? "(" + numErrors + " ERRORS!)" : "");
  }

  // ** how much time has elapsed since the last poll?
  var elapsed;
  {
    var now = new Date();
    elapsed = now - lastPoll;
    lastPoll = now;
  }

  // ** update running averages **
  updateAverages(elapsed);

  // ** determine how many activities to start **

  // how many active users would we like to simulate
  var targetActive = args.m;

  // if we're not throttled, then we'll trying 150% as many as
  // we're simulating right now.  If we're not simulating at least
  // 10000 active users, that shall be our lower bound
  if (!targetActive) {
    if (averages[0] > 10000) targetActive = averages[0] * 1.5;
    else targetActive = 10000;
  }

  // now how many new activities do we want to start?
  var newAct = activitiesPerUserPerSecond * targetActive;

  // scale based on how much time has elapsed since the last poll
  // on every iteration except the first
  if (iterations) newAct *= (elapsed / 1000);

  // probabilistic rounding
  {
    var add = (newAct % 1.0) < Math.random() ? 0 : 1;
    newAct = Math.floor(newAct) + add;
  }

  // ** start activities **

  // start the new activites until they're all started, or until we've
  // got twice as many outstanding as would be required by the target we
  // want to hit (which means the server can't keep up).
  while (newAct >= 1.0 && numOutstanding() < (activitiesPerUserPerSecond * targetActive * 2)) {
    numStarted++;
    startNewActivity();
    newAct--;
  }

  // ** schedule another wake up
  var wakeUpIn = 1000 - (new Date() - lastPoll);
  setTimeout(poll, wakeUpIn);

  // display averages
  outputAverages();

  iterations++;
}

// always start out by creating a bunch of users
var NUM_INITIAL_USERS = 100;

// if an explicit target was specified, let's output what that means
// in understandable terms
if (args.m) outputActiveUserSummary(args.m);

const userdb = require("../lib/load_gen/user_db.js");

if (args.u) {
  // parse args.u
  var start, end;
  try {
    var r = args.u.split('/');
    if (r.length != 2) throw "expected format ##/##";
    start = parseInt(r[0], 10);
    end = parseInt(r[1], 10);
    if (start >= end) throw "first number must be smaller than the second";
  } catch(e) {
    console.log("your -u argument is poorly formated: " + e.toString());
    process.exit(1);
  }

  // now create all them users!
  console.log("Runing with", (end - start + 1), "pre-created users (XXX@loadtest.domain)");
  for (var i = start; i < end; i++) {
    var user = userdb.getNewUser(i + "@loadtest.domain", "THE PASSWORD");
    userdb.releaseUser(user);
  }
  console.log("users created!  applying load...");
  poll();
} else {
  console.log("To start, let's create " + NUM_INITIAL_USERS + " users via the API.  One moment please...");

  var createUser = require("../lib/load_gen/activities/signup.js").startFunc;
  var created = 0;
  for (var i = 0; i < NUM_INITIAL_USERS; i++) {
    createUser(configuration, function(err) {
      if (err) {
        console.log("failed to create initial users! tragedy!  run away!:", err);
        process.exit(1);
      }
      process.stdout.write(".");
      if (++created == NUM_INITIAL_USERS) {
        process.stdout.write("\n\n");
        console.log("Average active users simulated over the last 1s/5s/60s:");
        poll();
      }
    });
  }
}