diff --git a/lib/static_resources.js b/lib/static_resources.js index 859d3d88ca5f9497ee03fbdcdafcd4e3c41705c0..fad0b6cff8cb7e0ac77101646ef5062ca36aef03 100644 --- a/lib/static_resources.js +++ b/lib/static_resources.js @@ -98,7 +98,7 @@ var dialog_js = und.flatten([ '/dialog/start.js' ]]); -exports.resources = resources = { +exports.resources = { '/production/dialog.css': [ '/css/common.css', '/dialog/css/popup.css', @@ -108,10 +108,30 @@ exports.resources = resources = { '/css/common.css', '/css/style.css', '/css/m.css' + ], + '/production/communication_iframe.js': [ + '/lib/jquery-1.7.1.min.js', + '/lib/jschannel.js', + '/lib/winchan.js', + '/lib/underscore-min.js', + '/lib/vepbundle.js', + '/lib/hub.js', + '/shared/javascript-extensions.js', + '/shared/browserid.js', + '/shared/mediator.js', + '/shared/helpers.js', + '/shared/storage.js', + '/shared/xhr.js', + '/shared/network.js', + '/shared/user.js', + '/communication_iframe/start.js' + ], + '/production/include.js': [ + '/include_js/include.js' ] }; -resources[dialog_min_js] = dialog_js; -resources[browserid_min_js] = browserid_js; +exports.resources[dialog_min_js] = dialog_js; +exports.resources[browserid_min_js] = browserid_js; var replace = function(path, locale) { return path.replace(':locale', locale); }; @@ -127,7 +147,7 @@ var replace = function(path, locale) { return path.replace(':locale', locale); } */ exports.all = function(langs) { var res = {}; - for (var f in resources) { + for (var f in exports.resources) { langs.forEach(function (lang) { var l = i18n.localeFrom(lang); res[replace(f, l)] = getResources(f, l); @@ -136,13 +156,28 @@ exports.all = function(langs) { return res; }; +/** return an array of all minified resources given the set of lang */ +exports.minified = function(langs) { + var res = []; + Object.keys(exports.resources).forEach(function(resource) { + if (resource.indexOf(':locale') != -1) { + langs.forEach(function (lang) { + res.push(replace(resource, i18n.localeFrom(lang))); + }); + } else { + res.push([ resource, null ]); + } + }); + return res; +}; + /** * Get all resource urls for a specified resource based on the locale */ exports.getResources = getResources = function(path, locale) { var res = []; - if (resources[path]) { - resources[path].forEach(function(r) { + if (exports.resources[path]) { + exports.resources[path].forEach(function(r) { res.push(replace(r, locale)); }); } diff --git a/package.json b/package.json index a5fc6a2919e3bd6bb7b41adc813a623b16c337fa..8823925882dadd4dad84fb5725f3cca14fd0e249 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "node-gettext": "0.1.1", "node-statsd": "https://github.com/downloads/lloyd/node-statsd/3a73de.tgz", "nodemailer": "0.1.18", + "mkdirp": "0.3.0", "optimist": "0.2.8", "postprocess": "0.2.4", "semver": "1.0.12", diff --git a/scripts/browserid.spec b/scripts/browserid.spec index 6ed39bbfd82a3d7c4c310e7b9d8de75e05742771..a6789a2d6f231ee621d8d3ce2cf76a5ddba43b29 100644 --- a/scripts/browserid.spec +++ b/scripts/browserid.spec @@ -25,8 +25,7 @@ npm install export PATH=$PWD/node_modules/.bin:$PATH ./locale/compile-mo.sh locale/ ./locale/compile-json.sh locale/ resources/static/i18n/ -scripts/compress.sh -env CONFIG_FILES=$PWD/config/l10n-all.json scripts/compress-locales.sh +env CONFIG_FILES=$PWD/config/l10n-all.json scripts/compress rm -r resources/static/build resources/static/test echo "$GIT_REVISION" > resources/static/ver.txt echo "locale svn r$SVN_REVISION" >> resources/static/ver.txt diff --git a/scripts/compress b/scripts/compress new file mode 100755 index 0000000000000000000000000000000000000000..d01f05826e5a715d1180b5115cc8887f17903ab0 --- /dev/null +++ b/scripts/compress @@ -0,0 +1,59 @@ +#!/usr/bin/env node + +var +path = require('path') +resources = require('../lib/static_resources.js'), +config = require('../lib/configuration.js'), +i18n = require('../lib/i18n'), +mkdirp = require('mkdirp'), +computecluster = require('compute-cluster'); + +const staticPath = path.join(__dirname, '..', 'resources', 'static'); + +var langs = config.get('supported_languages'); + +var all = resources.all(langs); + +var cc = new computecluster({ + module: path.join(__dirname, 'compress-worker.js'), + max_backlog: -1 +}); + +// first and foremost we'll "generate templates" - which is to concatenate +// a bunch of ejs into a javascript file +// NOTE: env setting could be cleaned up here, this is like this to minimally +// change things during migration of compress{,-locales}.sh to javascript +process.env['BUILD_DIR'] = path.join(staticPath, "build"); +mkdirp.sync(process.env['BUILD_DIR']); +process.env['TEMPLATE_DIR'] = path.join(staticPath, "dialog", "views"); +require('./create_templates.js')(); + +var leftToBuild = Object.keys(all).length; +var errors = 0; + +Object.keys(all).forEach(function(resource) { + // in dev, '/shared/templates.js' creates an empty object and templates + // are fetched on demand. + // in prod '/build/templates.js' has all templates glommed into it, + // and is bundled into the Big Minified Piles Of Resources we ship. + // Here we sub the former with the latter. + var ix = all[resource].indexOf('/shared/templates.js'); + if (ix != -1) all[resource].splice(ix, 1, '/build/templates.js'); + + cc.enqueue({ + file: resource, + deps: all[resource], + staticPath: staticPath + }, function(err, r) { + if (err || r.error) { + console.log("failed to build", resource,":", err || r.error); + errors++; + } else { + console.log("built", resource, "in", r.time + "s" + (r.info ? " (" + r.info + ")" : "")); + } + if (--leftToBuild == 0) { + cc.exit(); + if (errors) process.exit(1); + } + }); +}); diff --git a/scripts/compress-locales.sh b/scripts/compress-locales.sh deleted file mode 100755 index d3a7c0ec72cf28f30df6fcb3e92cab69f87c6289..0000000000000000000000000000000000000000 --- a/scripts/compress-locales.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/sh -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# compress-locales.sh creates all artifacts from browserid-locale.rpm -# and depends on compress.sh having already run - - -cd $(dirname "$0")/.. - -export PATH=$PWD/node_modules/.bin:$PATH - - -UGLIFY=`which uglifyjs 2> /dev/null` -if [ ! -x "$UGLIFY" ]; then - echo "uglifyjs not found in your path. Have you npm installed lately?" - exit 1 -fi - -#set up the path of where all build resources go. -BUILD_PATH=`pwd`'/resources/static/build' -if [ ! -x "$BUILD_PATH" ]; then - echo "****Creating build JS/CSS directory.****" - mkdir $BUILD_PATH -fi - -#set up the path of where all production resources go. -PRODUCTION_PATH=`pwd`'/resources/static/production' -if [ ! -x "$PRODUCTION_PATH" ]; then - echo "****Creating production JS/CSS directory.****" - mkdir $PRODUCTION_PATH -fi - -set -e # exit on errors - -echo '' -echo '****Building dialog HTML, CSS, and JS****' -echo '' - -cd $BUILD_PATH -cd .. -pwd - -# produce the dialog js -locales=`../../scripts/production_locales` -echo "generating for the following locales:" -echo $locales - -for locale in $locales; do - mkdir -p $BUILD_PATH/$locale - mkdir -p $BUILD_PATH/../i18n/$locale - # Touch as the trigger locale doesn't really exist - touch $BUILD_PATH/../i18n/${locale}/client.json - cat lib/jquery-1.7.1.min.js lib/winchan.js lib/underscore-min.js lib/vepbundle.js lib/ejs.js shared/javascript-extensions.js i18n/${locale}/client.json shared/gettext.js shared/browserid.js lib/hub.js lib/dom-jquery.js lib/module.js lib/jschannel.js $BUILD_PATH/templates.js shared/renderer.js shared/class.js shared/mediator.js shared/tooltip.js shared/validation.js shared/helpers.js shared/screens.js shared/browser-support.js shared/wait-messages.js shared/error-messages.js shared/error-display.js shared/storage.js shared/xhr.js shared/network.js shared/provisioning.js shared/user.js shared/command.js shared/history.js shared/state_machine.js shared/modules/page_module.js shared/modules/xhr_delay.js shared/modules/xhr_disable_form.js shared/modules/cookie_check.js lib/urlparse.js dialog/resources/internal_api.js dialog/resources/helpers.js dialog/resources/state.js dialog/controllers/actions.js dialog/controllers/dialog.js dialog/controllers/authenticate.js dialog/controllers/forgot_password.js dialog/controllers/check_registration.js dialog/controllers/pick_email.js dialog/controllers/add_email.js dialog/controllers/required_email.js dialog/controllers/verify_primary_user.js dialog/controllers/provision_primary_user.js dialog/controllers/primary_user_provisioned.js dialog/controllers/email_chosen.js dialog/start.js > $BUILD_PATH/$locale/dialog.uncompressed.js -done - -echo '' -echo '****Building BrowserID.org HTML, CSS, and JS****' -echo '' - -#produce the main site js -for locale in $locales; do - cat lib/vepbundle.js lib/jquery-1.7.1.min.js lib/underscore-min.js lib/ejs.js shared/javascript-extensions.js i18n/${locale}/client.json shared/gettext.js shared/browserid.js lib/dom-jquery.js lib/module.js lib/jschannel.js lib/winchan.js lib/hub.js $BUILD_PATH/templates.js shared/renderer.js shared/class.js shared/mediator.js shared/tooltip.js shared/validation.js shared/helpers.js shared/screens.js shared/browser-support.js shared/wait-messages.js shared/error-messages.js shared/error-display.js shared/mediator.js shared/storage.js shared/xhr.js shared/network.js shared/provisioning.js shared/user.js shared/modules/page_module.js shared/modules/xhr_delay.js shared/modules/xhr_disable_form.js shared/modules/cookie_check.js pages/page_helpers.js pages/start.js pages/index.js pages/add_email_address.js pages/verify_email_address.js pages/forgot.js pages/manage_account.js pages/signin.js pages/signup.js > $BUILD_PATH/$locale/browserid.uncompressed.js -done - -echo '' -echo '****Compressing all JS, CSS****' -echo '' - -cd $PRODUCTION_PATH - -pwd -# minify the JS -$UGLIFY < $BUILD_PATH/include.uncompressed.js > include.js -for locale in $locales; do - mkdir -p $locale - $UGLIFY < $BUILD_PATH/$locale/dialog.uncompressed.js > $locale/dialog.js - $UGLIFY < $BUILD_PATH/$locale/browserid.uncompressed.js > $locale/browserid.js -done diff --git a/scripts/compress-worker.js b/scripts/compress-worker.js new file mode 100644 index 0000000000000000000000000000000000000000..c905b52858cae66dbb44da336280a7d37bbe889e --- /dev/null +++ b/scripts/compress-worker.js @@ -0,0 +1,87 @@ +const +fs = require('fs'), +jsp = require("uglify-js").parser, +pro = require("uglify-js").uglify, +uglifycss = require('uglifycss'), +mkdirp = require('mkdirp'), +path = require('path'); + +function compressResource(staticPath, name, files, cb) { + var orig_code = ""; + var info = undefined; + function writeFile(final_code) { + mkdirp(path.join(staticPath, path.dirname(name)), function (err) { + if (err) cb(err); + else { + fs.writeFile(path.join(staticPath, name), final_code, function(err) { + cb(err, info); + }); + }; + }); + } + + function compress() { + var final_code; + if (/\.js$/.test(name)) { + // compress javascript + var ast = jsp.parse(orig_code); // parse code and get the initial AST + ast = pro.ast_mangle(ast); // get a new AST with mangled names + ast = pro.ast_squeeze(ast); // get an AST with compression optimizations + final_code = pro.split_lines(pro.gen_code(ast), 32 * 1024); // compressed code here + } else if (/\.css$/.test(name)) { + // compress css + final_code = uglifycss.processString(orig_code); + } else { + return cb("can't determine content type: " + name); + } + writeFile(final_code); + } + + function readNext() { + if (files.length) { + var f = files.shift(); + fs.readFile(path.join(staticPath, f), function(err, data) { + if (err) cb(err); + else { + orig_code += data; + readNext(); + } + }); + } else { + compress(); + } + } + + function isBuildNeeded() { + // we'll check mtime on all files. if any is newer than the output file, + // build is needed + try { + var lastGen = fs.statSync(path.join(staticPath, name)).mtime; + for (var i = 0; i < files.length; i++) { + if (lastGen < fs.statSync(path.join(staticPath, files[i])).mtime) { + info = "rebuilt because " + files[i] + " was changed"; + throw "newer"; + } + }; + // no rebuild needed + cb(null, "up to date"); + } catch (e) { + readNext(); + } + + } + + isBuildNeeded(); +} + +process.on('message', function(m) { + var startTime = new Date; + + compressResource(m.staticPath, m.file, m.deps, function(err, info) { + if (err) process.send({ error: err }); + else process.send({ + time: ((new Date - startTime) / 1000.0).toFixed(2), + info: info + }); + }); +}); \ No newline at end of file diff --git a/scripts/compress.sh b/scripts/compress.sh deleted file mode 100755 index d8d669f9bb5797e5e1470f0153d7b53afadc91c4..0000000000000000000000000000000000000000 --- a/scripts/compress.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/sh -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# compress.sh creates all artifacts from browserid.rpm - - -cd $(dirname "$0")/.. - -export PATH=$PWD/node_modules/.bin:$PATH - - -UGLIFY=`which uglifyjs 2> /dev/null` -if [ ! -x "$UGLIFY" ]; then - echo "uglifyjs not found in your path. Have you npm installed lately?" - exit 1 -fi - -UGLIFYCSS=`which uglifycss 2> /dev/null` -if [ ! -x "$UGLIFYCSS" ]; then - echo "uglifycss not found in your path. Have you npm installed lately?" - exit 1 -fi - -UGLIFYCSS=`pwd`'/node_modules/uglifycss/uglifycss' - -#set up the path of where all build resources go. -BUILD_PATH=`pwd`'/resources/static/build' -if [ ! -x "$BUILD_PATH" ]; then - echo "****Creating build JS/CSS directory.****" - mkdir $BUILD_PATH -fi - -#set up the path of where all production resources go. -PRODUCTION_PATH=`pwd`'/resources/static/production' -if [ ! -x "$PRODUCTION_PATH" ]; then - echo "****Creating production JS/CSS directory.****" - mkdir $PRODUCTION_PATH -fi - -set -e # exit on errors - -echo '' -echo '****Copy include.js****' -echo '' -cd resources/static -cp include_js/include.js $BUILD_PATH/include.uncompressed.js - -echo '' -echo '****Building dialog HTML, CSS, and JS****' -echo '' - -## This creates a combined templates file which is copied into -## resources/templates.js and included into the minified bundle. - -cd dialog/views -`BUILD_DIR=$BUILD_PATH ../../../../scripts/create_templates.js` -cd ../.. - -# produce the dialog css -cat css/common.css dialog/css/popup.css dialog/css/m.css > $BUILD_PATH/dialog.uncompressed.css - -# produce the non interactive frame js -cat lib/jquery-1.7.1.min.js lib/jschannel.js lib/winchan.js lib/underscore-min.js lib/vepbundle.js lib/hub.js shared/javascript-extensions.js shared/browserid.js shared/mediator.js shared/helpers.js shared/storage.js shared/xhr.js shared/network.js shared/user.js communication_iframe/start.js > $BUILD_PATH/communication_iframe.uncompressed.js - -echo '' -echo '****Building BrowserID.org CSS****' -echo '' - -# produce the main site css -cat css/common.css css/style.css css/m.css > $BUILD_PATH/browserid.uncompressed.css - -echo '' -echo '****Compressing all JS, CSS****' -echo '' - -cd $PRODUCTION_PATH - -pwd -$UGLIFY < $BUILD_PATH/communication_iframe.uncompressed.js > communication_iframe.js - - -# minify the CSS -$UGLIFYCSS $BUILD_PATH/browserid.uncompressed.css > browserid.css -$UGLIFYCSS $BUILD_PATH/dialog.uncompressed.css > dialog.css diff --git a/scripts/create_templates.js b/scripts/create_templates.js index 8d084ad68583d15b19bd4457008af1ef4d5a1a9b..4613cc7f8e0970f4cb9b868402e83cacc93ed54f 100755 --- a/scripts/create_templates.js +++ b/scripts/create_templates.js @@ -4,15 +4,33 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -const fs = require("fs"); +const +fs = require("fs"), +path = require('path'); var dir = process.env.TEMPLATE_DIR || process.cwd(); var output_dir = process.env.BUILD_DIR || dir; var templates = {}; +function generateTemplates() { + var fileNames = fs.readdirSync(dir) + + // is a regen even neccesary? + try { + var lastGen = fs.statSync(path.join(output_dir, "templates.js")).mtime; + for (var i = 0; i < fileNames.length; i++) { + if (lastGen < fs.statSync(path.join(dir, fileNames[i])).mtime) { + throw "newer"; + } + }; + // no rebuild needed + console.log("templates.js is up to date"); + return; + } catch (e) { + console.log("creating templates.js (" + e + ")"); + } -fs.readdir(dir, function(err, fileNames) { for(var index = 0, max = fileNames.length; index < max; index++) { var fileName = fileNames[index]; if(fileName.match(/\.ejs$/)) { @@ -24,5 +42,8 @@ fs.readdir(dir, function(err, fileNames) { var templateData = "BrowserID.Templates =" + JSON.stringify(templates) + ";"; fs.writeFileSync(output_dir + "/templates.js", templateData, "utf8"); -}); +}; +// run or export the function +if (process.argv[1] === __filename) generateTemplates(); +else module.exports = generateTemplates; diff --git a/scripts/deploy/vm.js b/scripts/deploy/vm.js index dd6d9b77699d7eacb06fce571d3c99f5a3a38fe6..c08d6949bd12ae8057577a1c89066440ed3f882f 100644 --- a/scripts/deploy/vm.js +++ b/scripts/deploy/vm.js @@ -4,7 +4,7 @@ jsel = require('JSONSelect'), key = require('./key.js'), sec = require('./sec.js'); -const BROWSERID_TEMPLATE_IMAGE_ID = 'ami-5678aa3f'; +const BROWSERID_TEMPLATE_IMAGE_ID = 'ami-7e954817'; function extractInstanceDeets(horribleBlob) { var instance = {}; diff --git a/scripts/production_locales b/scripts/production_locales deleted file mode 100755 index 7245ecbfea8ce0ae0f4ba1698ccf3c75734783d3..0000000000000000000000000000000000000000 --- a/scripts/production_locales +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env node -/* This is a helper script for compress.sh */ - -var path = require('path'); - -// configuration will create directories under VAR_PATH -process.env['VAR_PATH'] = '/tmp/browserid'; - -// Pick up production languages -process.env['CONFIG_FILES'] = process.env['CONFIG_FILES'] || path.join(__dirname, '..', 'config', 'local.json'); - -var path = require('path'), - format = require('util').format, - config = require(path.join(__dirname, '../lib/configuration.js')), - i18n = require(path.join(__dirname, '../lib/i18n.js')); - -var langs = config.get('supported_languages'); -process.stdout.write(format("%s\n", langs.map(i18n.localeFrom).join(' '))); diff --git a/tests/static-resource-test.js b/tests/static-resource-test.js index f7fa6603da31aec4b64ed4706383f3a8ed82d449..80fb7ebc0ced2a97b41f0929d4fe9037613bf3e7 100755 --- a/tests/static-resource-test.js +++ b/tests/static-resource-test.js @@ -23,7 +23,7 @@ suite.addBatch({ var res = resources.resources; assert.ok(files['/production/dialog.css'].length >= 3); // Get ride of non-localized asset bundles - ['/production/dialog.css', '/production/browserid.css'].forEach( + ['/production/communication_iframe.js', '/production/include.js', '/production/dialog.css', '/production/browserid.css'].forEach( function (nonLocaleAsset) { delete res[nonLocaleAsset]; delete files[nonLocaleAsset];