diff --git a/deploy.json b/deploy.json new file mode 100644 index 0000000..c50809b --- /dev/null +++ b/deploy.json @@ -0,0 +1,19 @@ +{ + "name": "appstore", + "targetPreparationScript": "./prepareDeployTarget.sh", + "hasSSL": false, + "hasGulp": false, + "hasPM2": true, + "environments": { + "prod": { + "tag": "prod", + "server": "ci.cloudron.io", + "remote": "ssh://ubuntu@ci.cloudron.io/home/ubuntu/app.git", + "ecosystem": "../keys/ci/ecosystem-prod.json", + "assets": { + "sshKey": "../keys/ci/id_rsa", + "sshPub": "../keys/ci/id_rsa.pub" + } + } + } +} diff --git a/e2etestrunner.js b/e2etestrunner.js new file mode 100755 index 0000000..44d8e1d --- /dev/null +++ b/e2etestrunner.js @@ -0,0 +1,119 @@ +/* jslint node:true */ + +'use strict'; + +require('./src/logger.js')(); +require('supererror')({ splatchError: true }); + +var async = require('async'), + debug = require('debug')('e2e:runner'), + fs = require('fs'), + path = require('path'), + shell = require('./shell.js'), + semver = require('semver'), + mailer = require('./mailer.js'), + superagent = require('superagent'), + util = require('util'); + +var E2E_TEST_DIR = path.join(process.env.HOME, 'e2e-test'); +var INSTALLER_DIR = path.join(process.env.HOME, 'installer'); + +// override debug.log to print only as console.log +debug.log = console.log.bind(console); + +var gLatestETag = null; // this allows us to rerun tests by restarting this process. and also for every appstore push +var gLatestBoxVersion = null; // for sending mail + +function start() { + runTestsIfNeeded(function () { + setTimeout(start, 60 * 1000); + }); +} + +function runTestsIfNeeded(callback) { + debug('Getting latest box version'); + + getLatestBoxVersion(function (error, latestETag, latestBoxVersion) { + if (error) return callback(error); + + if (latestETag === gLatestETag) { + debug('Box version has not changed. etag %s', gLatestETag); + return callback(null); + } + + debug('Box version has changed. etag %s', latestETag); + + runTests(latestETag, latestBoxVersion, function (error) { + debug('Finished running tests for etag %s: %s', latestETag, error); + gLatestETag = latestETag; + gLatestBoxVersion = latestBoxVersion; + callback(error); + }); + }); +} + +function getLatestBoxVersion(callback) { + superagent.get(process.env.BOX_VERSIONS_URL).end(function (error, res) { + if (error || res.statusCode !== 200 || !res.body) { + debug('Error downloading versions file', error, res.statusCode); + return callback(new Error('Error downloading versions file')); + } + + var latestVersion = Object.keys(res.body).sort(semver.rcompare)[0]; + var latestETag = res.headers['etag']; + + debug('%j:', res.headers); + + callback(null, latestETag, latestVersion); + }); +} + +function gitPullLatestTests(callback) { + async.series([ + function (next) { + if (fs.existsSync(E2E_TEST_DIR)) return next(); + debug('Cloning e2e-test repo on first run'); + shell.system('gitCloneE2E', 'git clone ' + process.env.E2E_TEST_REPO + ' e2e-test', + { cwd: process.env.HOME }, next); + }, + function (next) { + if (fs.existsSync(INSTALLER_DIR)) return next(); + debug('Cloning install repo on first run'); + shell.system('gitCloneInstaller', 'git clone ' + process.env.INSTALLER_REPO + ' installer', + { cwd: process.env.HOME }, next); + }, + function (next) { + debug('getting latest e2e-test'); + shell.system('gitFetchE2E', 'git fetch origin && git reset --hard origin/master && npm install', + { cwd: E2E_TEST_DIR, env: { GIT_DIR: E2E_TEST_DIR + '/.git' } }, next); // it's not clear why GIT_DIR is needed + }, + function (next) { + debug('getting latest installer'); + shell.system('gitFetchInstaller', 'git fetch origin && git reset --hard origin/master && npm install', + { cwd: INSTALLER_DIR, env: { GIT_DIR: INSTALLER_DIR + '/.git' } }, next); // it's not clear why GIT_DIR is needed + } + ], callback); +} + +function runTests(latestETag, latestBoxVersion, callback) { + var topic = util.format('box version: %s etag: %s', latestBoxVersion, latestETag); + + debug('Running tests for %s', topic); + + gitPullLatestTests(function (error, stdout, stderr) { + if (error) { + mailer.sendEndToEndTestResult(topic, stdout ? stdout.toString('utf8') : '', stderr ? stderr.toString('utf8') : '', function () { }); + return callback(error); + } + + shell.system('e2etestrunner', 'npm test', { cwd: E2E_TEST_DIR }, function (error, stdout, stderr) { + debug('Final test result', error); + + mailer.sendEndToEndTestResult(topic, stdout ? stdout.toString('utf8') : '', stderr ? stderr.toString('utf8') : '', function () { }); + return callback(error); + }); + }); +} + +debug('e2etest started.'); +start(); diff --git a/mailer.js b/mailer.js new file mode 100644 index 0000000..3cc41e5 --- /dev/null +++ b/mailer.js @@ -0,0 +1,50 @@ +/* jslint node: true */ + +'use strict'; + +var assert = require('assert'), + debug = require('debug')('e2e:mailer'), + postmark = require('postmark')(config.postmarkApiKey()), + util = require('util'); + +exports = module.exports = { + sendEndToEndTestResult: sendEndToEndTestResult +}; + +function send(options, callback) { + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof options.to, 'string'); + assert.strictEqual(typeof options.subject, 'string'); + assert.strictEqual(typeof options.text, 'string'); + + debug('Sending email to %s with subject "%s".', options.to, options.subject); + + postmark.send({ + 'From': options.from || 'no-reply@cloudron.io', + 'To': options.to, + 'Bcc': options.bcc, + 'Subject': options.subject, + 'TextBody': options.text, + 'HtmlBody': options.html, + 'Tag': 'Important' + }, function (error, success) { + if (error) { + console.error('Unable to send via postmark: ', error); + return callback(error); + } + + callback(); + }); +} + +function sendEndToEndTestResult(topic, stdout, stderr, callback) { + debug('Sending e2e test result for %s', topic); + + var mailOptions = { + to: 'admin@cloudron.io', + subject: util.format('E2E test results for box %s', topic), + text: 'stdout\n------\n' + stdout.toString('utf8') + '\n\nstderr\n------\n' + stderr.toString('utf8') + '\n\n' + }; + + send(mailOptions, callback); +} diff --git a/package.json b/package.json index 98728d0..90ea1be 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "colors": "^1.1.0", "debug": "^2.2.0", "mocha": "^2.2.5", + "once": "^1.3.2", "parallel-mocha": "0.0.7", "readline-sync": "^1.2.19", "semver": "^4.3.6", diff --git a/prepareDeployTarget.sh b/prepareDeployTarget.sh new file mode 100644 index 0000000..9ecce06 --- /dev/null +++ b/prepareDeployTarget.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +###################################### +## This script is run on the server ## +###################################### + +set -eux -o pipefail + +readonly APP_DIR="/home/ubuntu/app" +readonly CERTS_DIR="/home/ubuntu/certs" +readonly APP_REPO_DIR="/home/ubuntu/app.git" +readonly INIT_MASTER_DIR="/tmp/repoMasterSetup" +readonly ECOSYSTEM="/home/ubuntu/ecosystem.json" +readonly SERVER_NAME="<%= serverName %>" +readonly ENV="<%= env %>" + +swap_file="/swap" +[[ -f "${swap_file}" ]] && sudo swapoff "${swap_file}" +if [[ "${ENV}" == "dev" ]]; then + sudo fallocate -l 2048m "${swap_file}" # t1.micro has 1GB RAM +elif [[ "${ENV}" == "staging" ]]; then + sudo fallocate -l 2048m "${swap_file}" # t1.micro has 1GB RAM +else + sudo fallocate -l 2048m "${swap_file}" # t1.micro has 1GB RAM +fi +sudo chmod 600 "${swap_file}" +sudo mkswap "${swap_file}" +sudo swapon "${swap_file}" +if ! grep "${swap_file}" /etc/fstab; then + sudo bash -c "echo '${swap_file} none swap sw 0 0' >> /etc/fstab" +fi + +# install git +sudo apt-get update +sudo apt-get install -y git + +# install node v0.12 https://github.com/nodesource/distributions +sudo apt-get install -y python-software-properties python g++ make +curl -sL https://deb.nodesource.com/setup_0.12 | sudo bash - +sudo apt-get install -y nodejs + +# copy over ssh keys +sudo mv "/tmp/id_rsa" "${HOME}/.ssh/id_rsa" +sudo mv "/tmp/id_rsa.pub" "${HOME}/.ssh/id_rsa.pub" +echo -e "Host *.cloudron.me\n StrictHostKeyChecking no" > ${HOME}/.ssh/config + +# install tmpreaper (runs everyday and clean tmp) +sudo apt-get install -y tmpreaper +sudo sed -e 's/SHOWWARNING=true/# SHOWWARNING=true/' -i /etc/tmpreaper.conf + +# install pm2 +sudo npm install -g pm2 pm2-run forever +sudo rm -rf ~/.npm # .npm will get owned by root after "sudo npm install -g" + +# create deploy directories +mkdir -p "${APP_DIR}" + +# prepare git repo in case there is not yet one +if [[ ! -d "${APP_REPO_DIR}" ]]; then + mkdir -p "${APP_REPO_DIR}" + cd "${APP_REPO_DIR}" + git init --bare + + # create stub master branch + mkdir -p "${INIT_MASTER_DIR}" && cd "${INIT_MASTER_DIR}" + git init + touch stub + git add stub + git commit -m "stub" + git push ${APP_REPO_DIR} master --force + cd && rm -rf "${INIT_MASTER_DIR}" +else + echo "Git repo already exists at ${APP_REPO_DIR}" +fi + +# install post-receive hook +cat > "${APP_REPO_DIR}/hooks/post-receive" < --cmd "npm prune && npm install --production" +pm2 startOrRestart ${ECOSYSTEM} --env <%= env %> + +# save the new status for restart +pm2 save + +EOF +chmod +x "${APP_REPO_DIR}/hooks/post-receive" + + +# create empty ecosystem.json +touch "${ECOSYSTEM}" + +# run pm2 as current user ubuntu to avoid pm2 running as root below +pm2 status + +# ensure we run the latest version, this might not be the case if this script is ran against a already prepared server +pm2 updatePM2 + +# setup the init scripts +sudo pm2 startup -u ubuntu + diff --git a/shell.js b/shell.js new file mode 100644 index 0000000..9ab370f --- /dev/null +++ b/shell.js @@ -0,0 +1,95 @@ +'use strict'; + +exports = module.exports = { + sudo: sudo, + exec: exec, + system: system +}; + +var assert = require('assert'), + child_process = require('child_process'), + debug = require('debug')('e2e:shell.js'), + once = require('once'), + util = require('util'); + +var SUDO = '/usr/bin/sudo'; + +function exec(tag, file, args, options, callback) { + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof file, 'string'); + assert(util.isArray(args)); + + if (typeof options === 'function') { + callback = options; + options = { }; + } else { + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof callback, 'function'); + } + + callback = once(callback); // exit may or may not be called after an 'error' + + debug(tag + ' execFile: %s %s', file, args.join(' ')); + + var cp = child_process.spawn(file, args, options); + cp.stdout.on('data', function (data) { + debug(tag + ' (stdout): %s', data.toString('utf8')); + }); + + cp.stderr.on('data', function (data) { + debug(tag + ' (stderr): %s', data.toString('utf8')); + }); + + cp.on('exit', function (code, signal) { + if (code || signal) debug(tag + ' code: %s, signal: %s', code, signal); + + callback(code === 0 ? null : new Error(util.format('Exited with error %s signal %s', code, signal))); + }); + + cp.on('error', function (error) { + debug(tag + ' code: %s, signal: %s', error.code, error.signal); + callback(error); + }); + + return cp; +} + +function system(tag, cmd, options, callback) { + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof cmd, 'string'); + + if (typeof options === 'function') { + callback = options; + options = { }; + } else { + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof callback, 'function'); + } + + debug(tag + ' system: %s %j', cmd, options); + + var cp = child_process.exec(cmd, options, function (error, stdout, stderr) { + if (error) debug(tag + ' (error) ' + error); + + callback(error, stdout, stderr); + }); + + cp.stdout.on('data', function (data) { + debug(tag + ' (stdout): %s', data.toString('utf8')); + }); + + cp.stderr.on('data', function (data) { + debug(tag + ' (stderr): %s', data.toString('utf8')); + }); +} + +function sudo(tag, args, callback) { + assert.strictEqual(typeof tag, 'string'); + assert(util.isArray(args)); + assert.strictEqual(typeof callback, 'function'); + + // -S makes sudo read stdin for password + var cp = exec(tag, SUDO, [ '-S' ].concat(args), callback); + cp.stdin.end(); +} +