diff --git a/.awsbox.json b/.awsbox.json new file mode 100644 index 0000000000000000000000000000000000000000..ea13af1bd51f50978c01009404f6cf9668ba5b2a --- /dev/null +++ b/.awsbox.json @@ -0,0 +1,23 @@ +{ + "processes": [ + "bin/router", + "bin/proxy", + "bin/dbwriter", + "bin/keysigner", + "bin/verifier", + "bin/browserid" + ], + "env": { + "CONFIG_FILES": "$HOME/code/config/production.json,$HOME/code/config/aws.json,$HOME/config.json" + }, + "remote_hooks": { + "postdeploy": "scripts/awsbox/post_deploy.js", + "poststart": "scripts/show_config.js" + }, + "local_hooks": { + "postcreate": "scripts/awsbox/post_create.js" + }, + "packages": [ + "mysql-server" + ] +} diff --git a/package.json b/package.json index c9412e26b9793e18d97d4b6db58b3eaf646723e5..952fd2d94dce76f12b2e80aaa90d2e4dbe76e0e4 100644 --- a/package.json +++ b/package.json @@ -35,9 +35,8 @@ "winston": "0.5.6" }, "devDependencies": { - "xml2js": "0.1.13", "vows": "0.5.13", - "aws-lib": "0.0.5", + "awsbox": "0.0.10", "irc": "0.3.3" }, "scripts": { diff --git a/scripts/awsbox/post_deploy.js b/scripts/awsbox/post_deploy.js new file mode 100755 index 0000000000000000000000000000000000000000..af133aa60c7652b91cdc1bf66b2a616af507a92c --- /dev/null +++ b/scripts/awsbox/post_deploy.js @@ -0,0 +1,16 @@ +#!/bin/bash + +if [ ! -f $HOME/var/root.cert ] ; then + echo ">> generating keypair" + scripts/generate_ephemeral_keys.sh + mv var/root.{cert,secretkey} $HOME/var +else + echo ">> no keypair needed. you gots one" +fi + +echo ">> updating strings" +svn co -q http://svn.mozilla.org/projects/l10n-misc/trunk/browserid/locale +./locale/compile-mo.sh locale/ + +echo ">> generating production resources" +scripts/compress diff --git a/scripts/deploy.js b/scripts/deploy.js deleted file mode 100755 index aebb6c72ff31fdd28b0bc0a31e197f44d53f68b2..0000000000000000000000000000000000000000 --- a/scripts/deploy.js +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env node - -const -aws = require('./deploy/aws.js'); -path = require('path'); -vm = require('./deploy/vm.js'), -key = require('./deploy/key.js'), -ssh = require('./deploy/ssh.js'), -git = require('./deploy/git.js'), -dns = require('./deploy/dns.js'); - -var verbs = {}; - -function checkErr(err) { - if (err) { - process.stderr.write('fatal error: ' + err + "\n"); - process.exit(1); - } -} - -function printInstructions(name, deets) { - console.log("Yay! You have your very own deployment. Here's the basics:\n"); - console.log(" 1. deploy your code: git push " + name + " <mybranch>:master"); - console.log(" 2. visit your server on the web: https://" + name + ".hacksign.in"); - console.log(" 3. test via a website: http://" + name + ".myfavoritebeer.org"); - console.log(" 4. ssh in with sudo: ssh ec2-user@" + name + ".hacksign.in"); - console.log(" 5. ssh as the deployment user: ssh app@" + name + ".hacksign.in\n"); - console.log("enjoy! Here's your server details", JSON.stringify(deets, null, 4)); -} - -function validateName(name) { - if (!/^[a-z][0-9a-z_\-]*$/.test(name)) { - throw "invalid name! must be a valid dns fragment ([z-a0-9\-_])"; - } -} - -verbs['destroy'] = function(args) { - if (!args || args.length != 1) { - throw 'missing required argument: name of instance'; - } - var name = args[0]; - validateName(name); - var hostname = name + ".hacksign.in"; - - process.stdout.write("trying to destroy VM for " + hostname + ": "); - vm.destroy(name, function(err, deets) { - console.log(err ? ("failed: " + err) : "done"); - process.stdout.write("trying to remove DNS for " + hostname + ": "); - dns.deleteRecord(hostname, function(err) { - console.log(err ? "failed: " + err : "done"); - if (deets && deets.ipAddress) { - process.stdout.write("trying to remove git remote: "); - git.removeRemote(name, deets.ipAddress, function(err) { - console.log(err ? "failed: " + err : "done"); - }); - } - }); - }); -} - -verbs['test'] = function() { - // let's see if we can contact aws and zerigo - process.stdout.write("Checking DNS management access: "); - dns.inUse("somerandomname", function(err) { - console.log(err ? "NOT ok: " + err : "good"); - process.stdout.write("Checking AWS access: "); - vm.list(function(err) { - console.log(err ? "NOT ok: " + err : "good"); - }); - }); -} - -verbs['deploy'] = function(args) { - if (!args || args.length != 1) { - throw 'missing required argument: name of instance'; - } - var name = args[0]; - validateName(name); - var hostname = name + ".hacksign.in"; - var longName = 'browserid deployment (' + name + ')'; - - console.log("attempting to set up " + name + ".hacksign.in"); - - dns.inUse(hostname, function(err, r) { - checkErr(err); - if (r) checkErr("sorry! that name '" + name + "' is already being used. so sad"); - - vm.startImage(function(err, r) { - checkErr(err); - console.log(" ... VM launched, waiting for startup (should take about 20s)"); - - vm.waitForInstance(r.instanceId, function(err, deets) { - checkErr(err); - console.log(" ... Instance ready, setting up DNS"); - dns.updateRecord(name, "hacksign.in", deets.ipAddress, function(err) { - checkErr(err); - console.log(" ... DNS set up, setting human readable name in aws"); - - vm.setName(r.instanceId, longName, function(err) { - checkErr(err); - console.log(" ... name set, waiting for ssh access and configuring"); - var config = { public_url: "https://" + name + ".hacksign.in"}; - - ssh.copyUpConfig(deets.ipAddress, config, function(err, r) { - checkErr(err); - console.log(" ... victory! server is accessible and configured"); - git.addRemote(name, deets.ipAddress, function(err, r) { - if (err && /already exists/.test(err)) { - console.log("OOPS! you already have a git remote named 'test'!"); - console.log("to create a new one: git remote add <name> " + - "app@" + deets.ipAddress + ":git"); - } else { - checkErr(err); - } - console.log(" ... and your git remote is all set up"); - console.log(""); - printInstructions(name, deets); - }); - }); - }); - }); - }); - }); - }); -}; - -verbs['list'] = function(args) { - vm.list(function(err, r) { - checkErr(err); - console.log(JSON.stringify(r, null, 2)); - }); -}; - -var error = (process.argv.length <= 2); - -if (!error) { - var verb = process.argv[2]; - if (!verbs[verb]) error = "no such command: " + verb; - else { - try { - verbs[verb](process.argv.slice(3)); - } catch(e) { - error = "error running '" + verb + "' command: " + e; - } - } -} - -if (error) { - if (typeof error === 'string') process.stderr.write('fatal error: ' + error + "\n\n"); - - process.stderr.write('A command line tool to deploy BrowserID onto Amazon\'s EC2\n'); - process.stderr.write('Usage: ' + path.basename(__filename) + - ' <' + Object.keys(verbs).join('|') + "> [args]\n"); - process.exit(1); -} diff --git a/scripts/deploy/aws.js b/scripts/deploy/aws.js deleted file mode 100644 index 6641989e9a8dc348d0b4d843d4c594386ec64a91..0000000000000000000000000000000000000000 --- a/scripts/deploy/aws.js +++ /dev/null @@ -1,7 +0,0 @@ -const -awslib = require('aws-lib'); - -module.exports = awslib.createEC2Client(process.env['AWS_ID'], process.env['AWS_SECRET'], { - version: '2011-12-15' -}); - diff --git a/scripts/deploy/dns.js b/scripts/deploy/dns.js deleted file mode 100644 index 99a2f588788e117ce87958508177ec52155e4ce0..0000000000000000000000000000000000000000 --- a/scripts/deploy/dns.js +++ /dev/null @@ -1,83 +0,0 @@ -const -http = require('http'), -xml2js = new (require('xml2js')).Parser(), -jsel = require('JSONSelect'); - -const envVar = 'BROWSERID_DEPLOY_DNS_KEY'; -if (!process.env[envVar]) { - throw "Missing api key! contact lloyd and set the key in your env: " - + envVar; -} - -const api_key = process.env[envVar]; - -function doRequest(method, path, body, cb) { - var req = http.request({ - auth: 'lloyd@hilaiel.com:' + api_key, - host: 'ns.zerigo.com', - port: 80, - path: path, - method: method, - headers: { - 'Content-Type': 'application/xml', - 'Content-Length': body ? body.length : 0 - } - }, function(r) { - if ((r.statusCode / 100).toFixed(0) != 2 && - r.statusCode != 404) { - return cb("non 200 status: " + r.statusCode); - } - buf = ""; - r.on('data', function(chunk) { - buf += chunk; - }); - r.on('end', function() { - xml2js.parseString(buf, cb); - }); - }); - if (body) req.write(body); - req.end(); -}; - -exports.updateRecord = function (hostname, zone, ip, cb) { - doRequest('GET', '/api/1.1/zones.xml', null, function(err, r) { - if (err) return cb(err); - var m = jsel.match('object:has(:root > .domain:val(?)) > .id .#', - [ zone ], r); - if (m.length != 1) return cb("couldn't extract domain id from zerigo"); - var path = '/api/1.1/hosts.xml?zone_id=' + m[0]; - var body = '<host><data>' + ip + '</data><host-type>A</host-type>'; - body += '<hostname>' + hostname + '</hostname>' - body += '</host>'; - doRequest('POST', path, body, function(err, r) { - cb(err); - }); - }); -}; - -exports.deleteRecord = function (hostname, cb) { - doRequest('GET', '/api/1.1/hosts.xml?fqdn=' + hostname, null, function(err, r) { - if (err) return cb(err); - var m = jsel.match('.host .id > .#', r); - if (!m.length) return cb("no such DNS record"); - function deleteOne() { - if (!m.length) return cb(null); - var one = m.shift(); - doRequest('DELETE', '/api/1.1/hosts/' + one + '.xml', null, function(err) { - if (err) return cb(err); - deleteOne(); - }); - } - deleteOne(); - }); -}; - -exports.inUse = function (hostname, cb) { - doRequest('GET', '/api/1.1/hosts.xml?fqdn=' + hostname, null, function(err, r) { - if (err) return cb(err); - var m = jsel.match('.host', r); - // we shouldn't have multiple! oops! let's return the first one - if (m.length) return cb(null, m[0]); - cb(null, null); - }); -} diff --git a/scripts/deploy/git.js b/scripts/deploy/git.js deleted file mode 100644 index 4fc20d10a31307ca53abf50ac0b963c6afb15e74..0000000000000000000000000000000000000000 --- a/scripts/deploy/git.js +++ /dev/null @@ -1,120 +0,0 @@ -const -child_process = require('child_process'); -spawn = child_process.spawn, -path = require('path'); - -exports.addRemote = function(name, host, cb) { - var cmd = 'git remote add ' + name + ' app@'+ host + ':git'; - child_process.exec(cmd, cb); -}; - -// remove a remote, but only if it is pointed to a specific -// host. This will keep deploy from killing manuall remotes -// that you've set up -exports.removeRemote = function(name, host, cb) { - var desired = 'app@'+ host + ':git'; - var cmd = 'git remote -v show | grep push'; - child_process.exec(cmd, function(err, r) { - try { - var remotes = {}; - r.split('\n').forEach(function(line) { - if (!line.length) return; - var line = line.split('\t'); - if (!line.length == 2) return; - remotes[line[0]] = line[1].split(" ")[0]; - }); - if (remotes[name] && remotes[name] === desired) { - child_process.exec('git remote rm ' + name, cb); - } else { - throw "no such remote"; - } - } catch(e) { - cb(e); - } - }); -}; - -exports.currentSHA = function(dir, cb) { - if (typeof dir === 'function' && cb === undefined) { - cb = dir; - dir = path.join(__dirname, '..', '..'); - } - console.log(dir); - - var p = spawn('git', [ 'log', '--pretty=%h', '-1' ], { - env: { GIT_DIR: path.join(dir, ".git") } - }); - var buf = ""; - p.stdout.on('data', function(d) { - buf += d; - }); - p.on('exit', function(code, signal) { - console.log(buf); - var gitsha = buf.toString().trim(); - if (gitsha && gitsha.length === 7) { - return cb(null, gitsha); - } - cb("can't extract git sha from " + dir); - }); -}; - -function splitAndEmit(chunk, cb) { - if (chunk) chunk = chunk.toString(); - if (typeof chunk === 'string') { - chunk.split('\n').forEach(function (line) { - line = line.trim(); - if (line.length) cb(line); - }); - } -} - -exports.push = function(dir, host, pr, cb) { - if (typeof host === 'function' && cb === undefined) { - cb = pr; - pr = host; - host = dir; - dir = path.join(__dirname, '..', '..'); - } - - var p = spawn('git', [ 'push', 'app@' + host + ":git", 'dev:master' ], { - env: { - GIT_DIR: path.join(dir, ".git"), - GIT_WORK_TREE: dir - } - }); - p.stdout.on('data', function(c) { splitAndEmit(c, pr); }); - p.stderr.on('data', function(c) { splitAndEmit(c, pr); }); - p.on('exit', function(code, signal) { - return cb(code = 0); - }); -}; - -exports.pull = function(dir, remote, branch, pr, cb) { - var p = spawn('git', [ 'pull', "-f", remote, branch + ":" + branch ], { - env: { - GIT_DIR: path.join(dir, ".git"), - GIT_WORK_TREE: dir, - PWD: dir - }, - cwd: dir - }); - - p.stdout.on('data', function(c) { splitAndEmit(c, pr); }); - p.stderr.on('data', function(c) { splitAndEmit(c, pr); }); - - p.on('exit', function(code, signal) { - return cb(code = 0); - }); -} - -exports.init = function(dir, cb) { - var p = spawn('git', [ 'init' ], { - env: { - GIT_DIR: path.join(dir, ".git"), - GIT_WORK_TREE: dir - } - }); - p.on('exit', function(code, signal) { - return cb(code = 0); - }); -}; diff --git a/scripts/deploy/key.js b/scripts/deploy/key.js deleted file mode 100644 index d93da0158159b5d43f7361189c7fda12381e97ae..0000000000000000000000000000000000000000 --- a/scripts/deploy/key.js +++ /dev/null @@ -1,57 +0,0 @@ -const -aws = require('./aws.js'), -path = require('path'), -fs = require('fs'), -child_process = require('child_process'), -jsel = require('JSONSelect'), -crypto = require('crypto'); - -const keyPath = process.env['PUBKEY'] || path.join(process.env['HOME'], ".ssh", "id_rsa.pub"); - -exports.read = function(cb) { - fs.readFile(keyPath, cb); -}; - -exports.fingerprint = function(cb) { - exports.read(function(err, buf) { - if (err) return cb(err); - var b = new Buffer(buf.toString().split(' ')[1], 'base64'); - var md5sum = crypto.createHash('md5'); - md5sum.update(b); - cb(null, md5sum.digest('hex')); - }); -/* - child_process.exec( - "ssh-keygen -lf " + keyPath, - function(err, r) { - if (!err) r = r.split(' ')[1]; - cb(err, r); - }); -*/ -}; - -exports.getName = function(cb) { - exports.fingerprint(function(err, fingerprint) { - if (err) return cb(err); - - var keyName = "browserid deploy key (" + fingerprint + ")"; - - // is this fingerprint known? - aws.call('DescribeKeyPairs', {}, function(result) { - var found = jsel.match(":has(.keyName:val(?)) > .keyName", [ keyName ], result); - if (found.length) return cb(null, keyName); - - // key isn't yet installed! - exports.read(function(err, key) { - aws.call('ImportKeyPair', { - KeyName: keyName, - PublicKeyMaterial: new Buffer(key).toString('base64') - }, function(result) { - if (!result) return cb('null result from ec2 on key addition'); - if (result.Errors) return cb(result.Errors.Error.Message); - cb(null, keyName); - }); - }); - }); - }); -}; diff --git a/scripts/deploy/sec.js b/scripts/deploy/sec.js deleted file mode 100644 index d8211692677c1e0d3775f7abef89beffd3e6c737..0000000000000000000000000000000000000000 --- a/scripts/deploy/sec.js +++ /dev/null @@ -1,59 +0,0 @@ -const -aws = require('./aws.js'); -jsel = require('JSONSelect'), -key = require('./key.js'); - -// every time you change the security group, change this version number -// so new deployments will create a new group with the changes -const SECURITY_GROUP_VERSION = 1; - -function createError(msg, r) { - var m = jsel.match('.Message', r); - if (m.length) msg += ": " + m[0]; - return msg; -} - -exports.getName = function(cb) { - var groupName = "browserid group v" + SECURITY_GROUP_VERSION; - - // is this fingerprint known? - aws.call('DescribeSecurityGroups', { - GroupName: groupName - }, function(r) { - if (jsel.match('.Code:val("InvalidGroup.NotFound")', r).length) { - aws.call('CreateSecurityGroup', { - GroupName: groupName, - GroupDescription: 'A security group for browserid deployments' - }, function(r) { - if (!r || !r.return === 'true') { - return cb(createError('failed to create security group', r)); - } - aws.call('AuthorizeSecurityGroupIngress', { - GroupName: groupName, - "IpPermissions.1.IpProtocol": 'tcp', - "IpPermissions.1.FromPort": 80, - "IpPermissions.1.ToPort": 80, - "IpPermissions.1.IpRanges.1.CidrIp": "0.0.0.0/0", - "IpPermissions.2.IpProtocol": 'tcp', - "IpPermissions.2.FromPort": 22, - "IpPermissions.2.ToPort": 22, - "IpPermissions.2.IpRanges.1.CidrIp": "0.0.0.0/0", - "IpPermissions.3.IpProtocol": 'tcp', - "IpPermissions.3.FromPort": 443, - "IpPermissions.3.ToPort": 443, - "IpPermissions.3.IpRanges.1.CidrIp" : "0.0.0.0/0" - }, function(r) { - if (!r || !r.return === 'true') { - return cb(createError('failed to create security group', r)); - } - cb(null, groupName); - }); - }); - } else { - // already exists? - var m = jsel.match('.securityGroupInfo > .item > .groupName', r); - if (m.length && m[0] === groupName) return cb(null, groupName); - cb(createError('error creating group', r)); - } - }); -}; diff --git a/scripts/deploy/ssh.js b/scripts/deploy/ssh.js deleted file mode 100644 index 290abf1d322745ef8f4fbc17eac90345f289e606..0000000000000000000000000000000000000000 --- a/scripts/deploy/ssh.js +++ /dev/null @@ -1,43 +0,0 @@ -const -child_process = require('child_process'), -temp = require('temp'), -fs = require('fs'); - -const MAX_TRIES = 20; - -exports.copyUpConfig = function(host, config, cb) { - var tries = 0; - temp.open({}, function(err, r) { - fs.writeFileSync(r.path, JSON.stringify(config, null, 4)); - var cmd = 'scp -o "StrictHostKeyChecking no" ' + r.path + ' app@' + host + ":config.json"; - function oneTry() { - child_process.exec(cmd, function(err, r) { - if (err) { - if (++tries > MAX_TRIES) return cb("can't connect via SSH. stupid amazon"); - console.log(" ... nope. not yet. retrying."); - setTimeout(oneTry, 5000); - } else { - cb(); - } - }); - } - oneTry(); - }); -}; - -exports.copySSL = function(host, pub, priv, cb) { - var cmd = 'scp -o "StrictHostKeyChecking no" ' + pub + ' ec2-user@' + host + ":/etc/ssl/certs/hacksign.in.crt"; - child_process.exec(cmd, function(err, r) { - if (err) return cb(err); - var cmd = 'scp -o "StrictHostKeyChecking no" ' + priv + ' ec2-user@' + host + ":/etc/ssl/certs/hacksign.in.key"; - child_process.exec(cmd, function(err, r) { - var cmd = 'ssh -o "StrictHostKeyChecking no" ec2-user@' + host + " 'sudo /etc/init.d/nginx restart'"; - child_process.exec(cmd, cb); - }); - }); -}; - -exports.addSSHPubKey = function(host, pubkey, cb) { - var cmd = 'ssh -o "StrictHostKeyChecking no" ec2-user@' + host + " 'echo \'" + pubkey + "\' >> .ssh/authorized_keys'"; - child_process.exec(cmd, cb); -}; diff --git a/scripts/deploy/vm.js b/scripts/deploy/vm.js deleted file mode 100644 index de38451d5b62993a03080d7d21da0d951d88099d..0000000000000000000000000000000000000000 --- a/scripts/deploy/vm.js +++ /dev/null @@ -1,121 +0,0 @@ -const -aws = require('./aws.js'); -jsel = require('JSONSelect'), -key = require('./key.js'), -sec = require('./sec.js'); - -const BROWSERID_TEMPLATE_IMAGE_ID = 'ami-6ed07107'; - -function extractInstanceDeets(horribleBlob) { - var instance = {}; - ["instanceId", "imageId", "instanceState", "dnsName", "keyName", "instanceType", - "ipAddress"].forEach(function(key) { - if (horribleBlob[key]) instance[key] = horribleBlob[key]; - }); - var name = jsel.match('.tagSet :has(.key:val("Name")) > .value', horribleBlob); - if (name.length) { - instance.fullName = name[0]; - // if this is a 'browserid deployment', we'll only display the hostname chosen by the - // user - var m = /^browserid deployment \((.*)\)$/.exec(instance.fullName); - instance.name = m ? m[1] : instance.fullName; - } else { - instance.name = instance.instanceId; - } - return instance; -} - -exports.list = function(cb) { - aws.call('DescribeInstances', {}, function(result) { - var instances = {}; - var i = 1; - jsel.forEach( - '.instancesSet > .item:has(.instanceState .name:val("running"))', - result, function(item) { - var deets = extractInstanceDeets(item); - instances[deets.name || 'unknown ' + i++] = deets; - }); - cb(null, instances); - }); -}; - -exports.destroy = function(name, cb) { - exports.list(function(err, r) { - if (err) return cb('failed to list vms: ' + err); - if (!r[name]) return cb('no such vm'); - - aws.call('TerminateInstances', { - InstanceId: r[name].instanceId - }, function(result) { - try { return cb(result.Errors.Error.Message); } catch(e) {}; - cb(null, r[name]); - }); - }); -}; - -function returnSingleImageInfo(result, cb) { - if (!result) return cb('no results from ec2 api'); - try { return cb(result.Errors.Error.Message); } catch(e) {}; - try { - result = jsel.match('.instancesSet > .item', result)[0]; - cb(null, extractInstanceDeets(result)); - } catch(e) { - return cb("couldn't extract new instance details from ec2 response: " + e); - } -} - -exports.startImage = function(cb) { - key.getName(function(err, keyName) { - if (err) return cb(err); - sec.getName(function(err, groupName) { - if (err) return cb(err); - aws.call('RunInstances', { - ImageId: BROWSERID_TEMPLATE_IMAGE_ID, - KeyName: keyName, - SecurityGroup: groupName, - InstanceType: 't1.micro', - MinCount: 1, - MaxCount: 1 - }, function (result) { - returnSingleImageInfo(result, cb); - }); - }); - }); -}; - -exports.waitForInstance = function(id, cb) { - aws.call('DescribeInstanceStatus', { - InstanceId: id - }, function(r) { - if (!r) return cb('no response from ec2'); - // we're waiting and amazon might not have created the image yet! that's - // not an error, just an api timing quirk - var waiting = jsel.match('.Error .Code:val("InvalidInstanceID.NotFound")', r); - if (waiting.length) { - return setTimeout(function(){ exports.waitForInstance(id, cb); }, 1000); - } - - if (!r.instanceStatusSet) return cb('malformed response from ec2' + JSON.stringify(r, null, 2)); - if (Object.keys(r.instanceStatusSet).length) { - var deets = extractInstanceDeets(r.instanceStatusSet.item); - if (deets && deets.instanceState && deets.instanceState.name === 'running') { - return aws.call('DescribeInstances', { InstanceId: id }, function(result) { - returnSingleImageInfo(result, cb); - }); - } - } - setTimeout(function(){ exports.waitForInstance(id, cb); }, 1000); - }); -}; - -exports.setName = function(id, name, cb) { - aws.call('CreateTags', { - "ResourceId.0": id, - "Tag.0.Key": 'Name', - "Tag.0.Value": name - }, function(result) { - if (result && result.return === 'true') return cb(null); - try { return cb(result.Errors.Error.Message); } catch(e) {}; - return cb('unknown error setting instance name'); - }); -};