From 3f53f8dda5e864ae93840124682083bb4ba759a3 Mon Sep 17 00:00:00 2001
From: Lloyd Hilaiel <lloyd@hilaiel.com>
Date: Thu, 1 Mar 2012 10:09:11 -0700
Subject: [PATCH] rewrite compress scripts in javascript: better output, build
 of resources only when needed, uses multiple    cores, and leverages the same
 resource lists as cachify so we only express what dev resources go into what
 prod resources once #DRY.  closes #1009 closes #660

---
 lib/static_resources.js       | 47 ++++++++++++++++---
 package.json                  |  1 +
 scripts/browserid.spec        |  3 +-
 scripts/compress              | 59 ++++++++++++++++++++++++
 scripts/compress-locales.sh   | 80 --------------------------------
 scripts/compress-worker.js    | 87 +++++++++++++++++++++++++++++++++++
 scripts/compress.sh           | 86 ----------------------------------
 scripts/create_templates.js   | 27 +++++++++--
 scripts/deploy/vm.js          |  2 +-
 scripts/production_locales    | 18 --------
 tests/static-resource-test.js |  2 +-
 11 files changed, 215 insertions(+), 197 deletions(-)
 create mode 100755 scripts/compress
 delete mode 100755 scripts/compress-locales.sh
 create mode 100644 scripts/compress-worker.js
 delete mode 100755 scripts/compress.sh
 delete mode 100755 scripts/production_locales

diff --git a/lib/static_resources.js b/lib/static_resources.js
index 859d3d88c..fad0b6cff 100644
--- a/lib/static_resources.js
+++ b/lib/static_resources.js
@@ -98,7 +98,7 @@ var dialog_js = und.flatten([
     '/dialog/start.js'
   ]]);
 
-exports.resources = resources = {
+exports.resources = {
   '/production/dialog.css': [
     '/css/common.css',
     '/dialog/css/popup.css',
@@ -108,10 +108,30 @@ exports.resources = resources = {
     '/css/common.css',
     '/css/style.css',
     '/css/m.css'
+  ],
+  '/production/communication_iframe.js': [
+    '/lib/jquery-1.7.1.min.js',
+    '/lib/jschannel.js',
+    '/lib/winchan.js',
+    '/lib/underscore-min.js',
+    '/lib/vepbundle.js',
+    '/lib/hub.js',
+    '/shared/javascript-extensions.js',
+    '/shared/browserid.js',
+    '/shared/mediator.js',
+    '/shared/helpers.js',
+    '/shared/storage.js',
+    '/shared/xhr.js',
+    '/shared/network.js',
+    '/shared/user.js',
+    '/communication_iframe/start.js'
+  ],
+  '/production/include.js': [
+    '/include_js/include.js'
   ]
 };
-resources[dialog_min_js] = dialog_js;
-resources[browserid_min_js] = browserid_js;
+exports.resources[dialog_min_js] = dialog_js;
+exports.resources[browserid_min_js] = browserid_js;
 
 var replace = function(path, locale) { return path.replace(':locale', locale); };
 
@@ -127,7 +147,7 @@ var replace = function(path, locale) { return path.replace(':locale', locale); }
  */
 exports.all = function(langs) {
   var res = {};
-  for (var f in resources) {
+  for (var f in exports.resources) {
     langs.forEach(function (lang) {
       var l = i18n.localeFrom(lang);
       res[replace(f, l)] = getResources(f, l);
@@ -136,13 +156,28 @@ exports.all = function(langs) {
   return res;
 };
 
+/** return an array of all minified resources given the set of lang */
+exports.minified = function(langs) {
+  var res = [];
+  Object.keys(exports.resources).forEach(function(resource) {
+    if (resource.indexOf(':locale') != -1) {
+      langs.forEach(function (lang) {
+        res.push(replace(resource, i18n.localeFrom(lang)));
+      });
+    } else {
+      res.push([ resource, null ]);
+    }
+  });
+  return res;
+};
+
 /**
  * Get all resource urls for a specified resource based on the locale
  */
 exports.getResources = getResources = function(path, locale) {
   var res = [];
-  if (resources[path]) {
-    resources[path].forEach(function(r) {
+  if (exports.resources[path]) {
+    exports.resources[path].forEach(function(r) {
       res.push(replace(r, locale));
     });
   }
diff --git a/package.json b/package.json
index a5fc6a291..882392588 100644
--- a/package.json
+++ b/package.json
@@ -22,6 +22,7 @@
         "node-gettext": "0.1.1",
         "node-statsd": "https://github.com/downloads/lloyd/node-statsd/3a73de.tgz",
         "nodemailer": "0.1.18",
+        "mkdirp": "0.3.0",
         "optimist": "0.2.8",
         "postprocess": "0.2.4",
         "semver": "1.0.12",
diff --git a/scripts/browserid.spec b/scripts/browserid.spec
index 6ed39bbfd..a6789a2d6 100644
--- a/scripts/browserid.spec
+++ b/scripts/browserid.spec
@@ -25,8 +25,7 @@ npm install
 export PATH=$PWD/node_modules/.bin:$PATH
 ./locale/compile-mo.sh locale/
 ./locale/compile-json.sh locale/ resources/static/i18n/
-scripts/compress.sh
-env CONFIG_FILES=$PWD/config/l10n-all.json scripts/compress-locales.sh
+env CONFIG_FILES=$PWD/config/l10n-all.json scripts/compress
 rm -r resources/static/build resources/static/test
 echo "$GIT_REVISION" > resources/static/ver.txt
 echo "locale svn r$SVN_REVISION" >> resources/static/ver.txt
diff --git a/scripts/compress b/scripts/compress
new file mode 100755
index 000000000..d01f05826
--- /dev/null
+++ b/scripts/compress
@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+var
+path = require('path')
+resources = require('../lib/static_resources.js'),
+config = require('../lib/configuration.js'),
+i18n = require('../lib/i18n'),
+mkdirp = require('mkdirp'),
+computecluster = require('compute-cluster');
+
+const staticPath = path.join(__dirname, '..', 'resources', 'static');
+
+var langs = config.get('supported_languages');
+
+var all = resources.all(langs);
+
+var cc = new computecluster({
+  module: path.join(__dirname, 'compress-worker.js'),
+  max_backlog: -1
+});
+
+// first and foremost we'll "generate templates" - which is to concatenate
+// a bunch of ejs into a javascript file
+// NOTE: env setting could be cleaned up here, this is like this to minimally
+// change things during migration of compress{,-locales}.sh to javascript
+process.env['BUILD_DIR'] = path.join(staticPath, "build");
+mkdirp.sync(process.env['BUILD_DIR']);
+process.env['TEMPLATE_DIR'] = path.join(staticPath, "dialog", "views");
+require('./create_templates.js')();
+
+var leftToBuild = Object.keys(all).length;
+var errors = 0;
+
+Object.keys(all).forEach(function(resource) {
+  // in dev, '/shared/templates.js' creates an empty object and templates
+  // are fetched on demand.
+  // in prod '/build/templates.js' has all templates glommed into it,
+  // and is bundled into the Big Minified Piles Of Resources we ship.
+  // Here we sub the former with the latter.
+  var ix = all[resource].indexOf('/shared/templates.js');
+  if (ix != -1) all[resource].splice(ix, 1, '/build/templates.js');
+
+  cc.enqueue({
+    file: resource,
+    deps: all[resource],
+    staticPath: staticPath
+  }, function(err, r) {
+    if (err || r.error) {
+      console.log("failed to build", resource,":", err || r.error);
+      errors++;
+    } else {
+      console.log("built", resource, "in", r.time + "s" + (r.info ? " (" + r.info + ")" : ""));
+    }
+    if (--leftToBuild == 0) {
+      cc.exit();
+      if (errors) process.exit(1);
+    }
+  });
+});
diff --git a/scripts/compress-locales.sh b/scripts/compress-locales.sh
deleted file mode 100755
index d3a7c0ec7..000000000
--- a/scripts/compress-locales.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/bin/sh
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# compress-locales.sh creates all artifacts from browserid-locale.rpm
-# and depends on compress.sh having already run
-
-
-cd $(dirname "$0")/..
-
-export PATH=$PWD/node_modules/.bin:$PATH
-
-
-UGLIFY=`which uglifyjs 2> /dev/null`
-if [ ! -x "$UGLIFY" ]; then
-    echo "uglifyjs not found in your path.  Have you npm installed lately?"
-    exit 1
-fi
-
-#set up the path of where all build resources go.
-BUILD_PATH=`pwd`'/resources/static/build'
-if [ ! -x "$BUILD_PATH" ]; then
-    echo "****Creating build JS/CSS directory.****"
-    mkdir $BUILD_PATH
-fi
-
-#set up the path of where all production resources go.
-PRODUCTION_PATH=`pwd`'/resources/static/production'
-if [ ! -x "$PRODUCTION_PATH" ]; then
-    echo "****Creating production JS/CSS directory.****"
-    mkdir $PRODUCTION_PATH
-fi
-
-set -e  # exit on errors
-
-echo ''
-echo '****Building dialog HTML, CSS, and JS****'
-echo ''
-
-cd $BUILD_PATH
-cd ..
-pwd
-
-# produce the dialog js
-locales=`../../scripts/production_locales`
-echo "generating for the following locales:"
-echo $locales
-
-for locale in $locales; do
-    mkdir -p $BUILD_PATH/$locale
-    mkdir -p $BUILD_PATH/../i18n/$locale
-    # Touch as the trigger locale doesn't really exist
-    touch $BUILD_PATH/../i18n/${locale}/client.json
-    cat lib/jquery-1.7.1.min.js lib/winchan.js lib/underscore-min.js lib/vepbundle.js lib/ejs.js shared/javascript-extensions.js i18n/${locale}/client.json shared/gettext.js shared/browserid.js lib/hub.js lib/dom-jquery.js lib/module.js lib/jschannel.js $BUILD_PATH/templates.js shared/renderer.js shared/class.js shared/mediator.js shared/tooltip.js shared/validation.js shared/helpers.js shared/screens.js shared/browser-support.js shared/wait-messages.js shared/error-messages.js shared/error-display.js shared/storage.js shared/xhr.js shared/network.js shared/provisioning.js shared/user.js shared/command.js shared/history.js shared/state_machine.js shared/modules/page_module.js shared/modules/xhr_delay.js shared/modules/xhr_disable_form.js shared/modules/cookie_check.js lib/urlparse.js dialog/resources/internal_api.js dialog/resources/helpers.js dialog/resources/state.js dialog/controllers/actions.js dialog/controllers/dialog.js dialog/controllers/authenticate.js dialog/controllers/forgot_password.js dialog/controllers/check_registration.js dialog/controllers/pick_email.js dialog/controllers/add_email.js dialog/controllers/required_email.js dialog/controllers/verify_primary_user.js dialog/controllers/provision_primary_user.js dialog/controllers/primary_user_provisioned.js dialog/controllers/email_chosen.js dialog/start.js > $BUILD_PATH/$locale/dialog.uncompressed.js
-done
-
-echo ''
-echo '****Building BrowserID.org HTML, CSS, and JS****'
-echo ''
-
-#produce the main site js
-for locale in $locales; do
-    cat lib/vepbundle.js lib/jquery-1.7.1.min.js lib/underscore-min.js lib/ejs.js shared/javascript-extensions.js i18n/${locale}/client.json shared/gettext.js shared/browserid.js lib/dom-jquery.js lib/module.js lib/jschannel.js lib/winchan.js lib/hub.js $BUILD_PATH/templates.js shared/renderer.js shared/class.js shared/mediator.js shared/tooltip.js shared/validation.js shared/helpers.js shared/screens.js shared/browser-support.js shared/wait-messages.js shared/error-messages.js shared/error-display.js shared/mediator.js shared/storage.js shared/xhr.js shared/network.js shared/provisioning.js shared/user.js shared/modules/page_module.js shared/modules/xhr_delay.js shared/modules/xhr_disable_form.js shared/modules/cookie_check.js pages/page_helpers.js pages/start.js pages/index.js pages/add_email_address.js pages/verify_email_address.js pages/forgot.js pages/manage_account.js pages/signin.js pages/signup.js > $BUILD_PATH/$locale/browserid.uncompressed.js
-done
-
-echo ''
-echo '****Compressing all JS, CSS****'
-echo ''
-
-cd $PRODUCTION_PATH
-
-pwd
-# minify the JS
-$UGLIFY < $BUILD_PATH/include.uncompressed.js > include.js
-for locale in $locales; do
-    mkdir -p $locale
-    $UGLIFY < $BUILD_PATH/$locale/dialog.uncompressed.js > $locale/dialog.js
-    $UGLIFY < $BUILD_PATH/$locale/browserid.uncompressed.js > $locale/browserid.js
-done
diff --git a/scripts/compress-worker.js b/scripts/compress-worker.js
new file mode 100644
index 000000000..c905b5285
--- /dev/null
+++ b/scripts/compress-worker.js
@@ -0,0 +1,87 @@
+const
+fs = require('fs'),
+jsp = require("uglify-js").parser,
+pro = require("uglify-js").uglify,
+uglifycss = require('uglifycss'),
+mkdirp = require('mkdirp'),
+path = require('path');
+
+function compressResource(staticPath, name, files, cb) {
+  var orig_code = "";
+  var info = undefined;
+  function writeFile(final_code) {
+    mkdirp(path.join(staticPath, path.dirname(name)), function (err) {
+      if (err) cb(err);
+      else {
+        fs.writeFile(path.join(staticPath, name), final_code, function(err) {
+          cb(err, info);
+        });
+      };
+    });
+  }
+
+  function compress() {
+    var final_code;
+    if (/\.js$/.test(name)) {
+      // compress javascript
+      var ast = jsp.parse(orig_code); // parse code and get the initial AST
+      ast = pro.ast_mangle(ast); // get a new AST with mangled names
+      ast = pro.ast_squeeze(ast); // get an AST with compression optimizations
+      final_code = pro.split_lines(pro.gen_code(ast), 32 * 1024); // compressed code here
+    } else if (/\.css$/.test(name)) {
+      // compress css
+      final_code = uglifycss.processString(orig_code);
+    } else {
+      return cb("can't determine content type: " + name);
+    }
+    writeFile(final_code);
+  }
+
+  function readNext() {
+    if (files.length) {
+      var f = files.shift();
+      fs.readFile(path.join(staticPath, f), function(err, data) {
+        if (err) cb(err);
+        else {
+          orig_code += data;
+          readNext();
+        }
+      });
+    } else {
+      compress();
+    }
+  }
+
+  function isBuildNeeded() {
+    // we'll check mtime on all files.  if any is newer than the output file,
+    // build is needed
+    try {
+      var lastGen = fs.statSync(path.join(staticPath, name)).mtime;
+      for (var i = 0; i < files.length; i++) {
+        if (lastGen < fs.statSync(path.join(staticPath, files[i])).mtime) {
+          info = "rebuilt because " + files[i] + " was changed";
+          throw "newer";
+        }
+      };
+      // no rebuild needed
+      cb(null, "up to date");
+    } catch (e) {
+      readNext();
+    }
+
+  }
+
+  isBuildNeeded();
+}
+
+process.on('message', function(m) {
+  var startTime = new Date;
+
+  compressResource(m.staticPath, m.file, m.deps, function(err, info) {
+    if (err) process.send({ error: err });
+    else process.send({
+      time: ((new Date - startTime) / 1000.0).toFixed(2),
+      info: info
+    });
+  });
+});
\ No newline at end of file
diff --git a/scripts/compress.sh b/scripts/compress.sh
deleted file mode 100755
index d8d669f9b..000000000
--- a/scripts/compress.sh
+++ /dev/null
@@ -1,86 +0,0 @@
-#!/bin/sh
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# compress.sh creates all artifacts from browserid.rpm
-
-
-cd $(dirname "$0")/..
-
-export PATH=$PWD/node_modules/.bin:$PATH
-
-
-UGLIFY=`which uglifyjs 2> /dev/null`
-if [ ! -x "$UGLIFY" ]; then
-    echo "uglifyjs not found in your path.  Have you npm installed lately?"
-    exit 1
-fi
-
-UGLIFYCSS=`which uglifycss 2> /dev/null`
-if [ ! -x "$UGLIFYCSS" ]; then
-    echo "uglifycss not found in your path.  Have you npm installed lately?"
-    exit 1
-fi
-
-UGLIFYCSS=`pwd`'/node_modules/uglifycss/uglifycss'
-
-#set up the path of where all build resources go.
-BUILD_PATH=`pwd`'/resources/static/build'
-if [ ! -x "$BUILD_PATH" ]; then
-    echo "****Creating build JS/CSS directory.****"
-    mkdir $BUILD_PATH
-fi
-
-#set up the path of where all production resources go.
-PRODUCTION_PATH=`pwd`'/resources/static/production'
-if [ ! -x "$PRODUCTION_PATH" ]; then
-    echo "****Creating production JS/CSS directory.****"
-    mkdir $PRODUCTION_PATH
-fi
-
-set -e  # exit on errors
-
-echo ''
-echo '****Copy include.js****'
-echo ''
-cd resources/static
-cp include_js/include.js $BUILD_PATH/include.uncompressed.js
-
-echo ''
-echo '****Building dialog HTML, CSS, and JS****'
-echo ''
-
-## This creates a combined templates file which is copied into
-## resources/templates.js and included into the minified bundle.
-
-cd dialog/views
-`BUILD_DIR=$BUILD_PATH ../../../../scripts/create_templates.js`
-cd ../..
-
-# produce the dialog css
-cat css/common.css dialog/css/popup.css dialog/css/m.css > $BUILD_PATH/dialog.uncompressed.css
-
-# produce the non interactive frame js
-cat lib/jquery-1.7.1.min.js lib/jschannel.js lib/winchan.js lib/underscore-min.js lib/vepbundle.js lib/hub.js shared/javascript-extensions.js shared/browserid.js shared/mediator.js shared/helpers.js shared/storage.js shared/xhr.js shared/network.js shared/user.js communication_iframe/start.js > $BUILD_PATH/communication_iframe.uncompressed.js
-
-echo ''
-echo '****Building BrowserID.org CSS****'
-echo ''
-
-# produce the main site css
-cat css/common.css css/style.css css/m.css > $BUILD_PATH/browserid.uncompressed.css
-
-echo ''
-echo '****Compressing all JS, CSS****'
-echo ''
-
-cd $PRODUCTION_PATH
-
-pwd
-$UGLIFY < $BUILD_PATH/communication_iframe.uncompressed.js > communication_iframe.js
-
-
-# minify the CSS
-$UGLIFYCSS $BUILD_PATH/browserid.uncompressed.css > browserid.css
-$UGLIFYCSS $BUILD_PATH/dialog.uncompressed.css > dialog.css
diff --git a/scripts/create_templates.js b/scripts/create_templates.js
index 8d084ad68..4613cc7f8 100755
--- a/scripts/create_templates.js
+++ b/scripts/create_templates.js
@@ -4,15 +4,33 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 
-const fs = require("fs");
+const
+fs = require("fs"),
+path = require('path');
 
 var dir = process.env.TEMPLATE_DIR || process.cwd();
 var output_dir = process.env.BUILD_DIR || dir;
 
 var templates = {};
 
+function generateTemplates() {
+  var fileNames = fs.readdirSync(dir)
+
+  // is a regen even neccesary?
+  try {
+    var lastGen = fs.statSync(path.join(output_dir, "templates.js")).mtime;
+    for (var i = 0; i < fileNames.length; i++) {
+      if (lastGen < fs.statSync(path.join(dir, fileNames[i])).mtime) {
+        throw "newer";
+      }
+    };
+    // no rebuild needed
+    console.log("templates.js is up to date");
+    return;
+  } catch (e) {
+    console.log("creating templates.js (" + e + ")");
+  }
 
-fs.readdir(dir, function(err, fileNames) {
   for(var index = 0, max = fileNames.length; index < max; index++) {
     var fileName = fileNames[index];
     if(fileName.match(/\.ejs$/)) {
@@ -24,5 +42,8 @@ fs.readdir(dir, function(err, fileNames) {
   var templateData = "BrowserID.Templates =" + JSON.stringify(templates) + ";";
 
   fs.writeFileSync(output_dir + "/templates.js", templateData, "utf8");
-});
+};
 
+// run or export the function
+if (process.argv[1] === __filename) generateTemplates();
+else module.exports = generateTemplates;
diff --git a/scripts/deploy/vm.js b/scripts/deploy/vm.js
index dd6d9b776..c08d6949b 100644
--- a/scripts/deploy/vm.js
+++ b/scripts/deploy/vm.js
@@ -4,7 +4,7 @@ jsel = require('JSONSelect'),
 key = require('./key.js'),
 sec = require('./sec.js');
 
-const BROWSERID_TEMPLATE_IMAGE_ID = 'ami-5678aa3f';
+const BROWSERID_TEMPLATE_IMAGE_ID = 'ami-7e954817';
 
 function extractInstanceDeets(horribleBlob) {
   var instance = {};
diff --git a/scripts/production_locales b/scripts/production_locales
deleted file mode 100755
index 7245ecbfe..000000000
--- a/scripts/production_locales
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env node
-/* This is a helper script for compress.sh */
-
-var path = require('path');
-
-// configuration will create directories under VAR_PATH
-process.env['VAR_PATH'] = '/tmp/browserid';
-
-// Pick up production languages
-process.env['CONFIG_FILES'] = process.env['CONFIG_FILES'] || path.join(__dirname, '..', 'config', 'local.json');
-
-var path = require('path'),
-    format = require('util').format,
-    config = require(path.join(__dirname, '../lib/configuration.js')),
-    i18n = require(path.join(__dirname, '../lib/i18n.js'));
-
-var langs = config.get('supported_languages');
-process.stdout.write(format("%s\n", langs.map(i18n.localeFrom).join(' ')));
diff --git a/tests/static-resource-test.js b/tests/static-resource-test.js
index f7fa6603d..80fb7ebc0 100755
--- a/tests/static-resource-test.js
+++ b/tests/static-resource-test.js
@@ -23,7 +23,7 @@ suite.addBatch({
       var res = resources.resources;
       assert.ok(files['/production/dialog.css'].length >= 3);
       // Get ride of non-localized asset bundles
-      ['/production/dialog.css', '/production/browserid.css'].forEach(
+      ['/production/communication_iframe.js', '/production/include.js', '/production/dialog.css', '/production/browserid.css'].forEach(
         function (nonLocaleAsset) {
           delete res[nonLocaleAsset];
           delete files[nonLocaleAsset];
-- 
GitLab