ci becomes it's own server
This commit is contained in:
parent
eafda6ce77
commit
e7ee9a29f3
19
deploy.json
Normal file
19
deploy.json
Normal file
@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "appstore",
|
||||
"targetPreparationScript": "./prepareDeployTarget.sh",
|
||||
"hasSSL": false,
|
||||
"hasGulp": false,
|
||||
"hasPM2": true,
|
||||
"environments": {
|
||||
"prod": {
|
||||
"tag": "prod",
|
||||
"server": "ci.cloudron.io",
|
||||
"remote": "ssh://ubuntu@ci.cloudron.io/home/ubuntu/app.git",
|
||||
"ecosystem": "../keys/ci/ecosystem-prod.json",
|
||||
"assets": {
|
||||
"sshKey": "../keys/ci/id_rsa",
|
||||
"sshPub": "../keys/ci/id_rsa.pub"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
119
e2etestrunner.js
Executable file
119
e2etestrunner.js
Executable file
@ -0,0 +1,119 @@
|
||||
/* jslint node:true */
|
||||
|
||||
'use strict';
|
||||
|
||||
require('./src/logger.js')();
|
||||
require('supererror')({ splatchError: true });
|
||||
|
||||
var async = require('async'),
|
||||
debug = require('debug')('e2e:runner'),
|
||||
fs = require('fs'),
|
||||
path = require('path'),
|
||||
shell = require('./shell.js'),
|
||||
semver = require('semver'),
|
||||
mailer = require('./mailer.js'),
|
||||
superagent = require('superagent'),
|
||||
util = require('util');
|
||||
|
||||
var E2E_TEST_DIR = path.join(process.env.HOME, 'e2e-test');
|
||||
var INSTALLER_DIR = path.join(process.env.HOME, 'installer');
|
||||
|
||||
// override debug.log to print only as console.log
|
||||
debug.log = console.log.bind(console);
|
||||
|
||||
var gLatestETag = null; // this allows us to rerun tests by restarting this process. and also for every appstore push
|
||||
var gLatestBoxVersion = null; // for sending mail
|
||||
|
||||
function start() {
|
||||
runTestsIfNeeded(function () {
|
||||
setTimeout(start, 60 * 1000);
|
||||
});
|
||||
}
|
||||
|
||||
function runTestsIfNeeded(callback) {
|
||||
debug('Getting latest box version');
|
||||
|
||||
getLatestBoxVersion(function (error, latestETag, latestBoxVersion) {
|
||||
if (error) return callback(error);
|
||||
|
||||
if (latestETag === gLatestETag) {
|
||||
debug('Box version has not changed. etag %s', gLatestETag);
|
||||
return callback(null);
|
||||
}
|
||||
|
||||
debug('Box version has changed. etag %s', latestETag);
|
||||
|
||||
runTests(latestETag, latestBoxVersion, function (error) {
|
||||
debug('Finished running tests for etag %s: %s', latestETag, error);
|
||||
gLatestETag = latestETag;
|
||||
gLatestBoxVersion = latestBoxVersion;
|
||||
callback(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function getLatestBoxVersion(callback) {
|
||||
superagent.get(process.env.BOX_VERSIONS_URL).end(function (error, res) {
|
||||
if (error || res.statusCode !== 200 || !res.body) {
|
||||
debug('Error downloading versions file', error, res.statusCode);
|
||||
return callback(new Error('Error downloading versions file'));
|
||||
}
|
||||
|
||||
var latestVersion = Object.keys(res.body).sort(semver.rcompare)[0];
|
||||
var latestETag = res.headers['etag'];
|
||||
|
||||
debug('%j:', res.headers);
|
||||
|
||||
callback(null, latestETag, latestVersion);
|
||||
});
|
||||
}
|
||||
|
||||
function gitPullLatestTests(callback) {
|
||||
async.series([
|
||||
function (next) {
|
||||
if (fs.existsSync(E2E_TEST_DIR)) return next();
|
||||
debug('Cloning e2e-test repo on first run');
|
||||
shell.system('gitCloneE2E', 'git clone ' + process.env.E2E_TEST_REPO + ' e2e-test',
|
||||
{ cwd: process.env.HOME }, next);
|
||||
},
|
||||
function (next) {
|
||||
if (fs.existsSync(INSTALLER_DIR)) return next();
|
||||
debug('Cloning install repo on first run');
|
||||
shell.system('gitCloneInstaller', 'git clone ' + process.env.INSTALLER_REPO + ' installer',
|
||||
{ cwd: process.env.HOME }, next);
|
||||
},
|
||||
function (next) {
|
||||
debug('getting latest e2e-test');
|
||||
shell.system('gitFetchE2E', 'git fetch origin && git reset --hard origin/master && npm install',
|
||||
{ cwd: E2E_TEST_DIR, env: { GIT_DIR: E2E_TEST_DIR + '/.git' } }, next); // it's not clear why GIT_DIR is needed
|
||||
},
|
||||
function (next) {
|
||||
debug('getting latest installer');
|
||||
shell.system('gitFetchInstaller', 'git fetch origin && git reset --hard origin/master && npm install',
|
||||
{ cwd: INSTALLER_DIR, env: { GIT_DIR: INSTALLER_DIR + '/.git' } }, next); // it's not clear why GIT_DIR is needed
|
||||
}
|
||||
], callback);
|
||||
}
|
||||
|
||||
function runTests(latestETag, latestBoxVersion, callback) {
|
||||
var topic = util.format('box version: %s etag: %s', latestBoxVersion, latestETag);
|
||||
|
||||
debug('Running tests for %s', topic);
|
||||
|
||||
gitPullLatestTests(function (error, stdout, stderr) {
|
||||
if (error) {
|
||||
mailer.sendEndToEndTestResult(topic, stdout ? stdout.toString('utf8') : '', stderr ? stderr.toString('utf8') : '', function () { });
|
||||
return callback(error);
|
||||
}
|
||||
|
||||
shell.system('e2etestrunner', 'npm test', { cwd: E2E_TEST_DIR }, function (error, stdout, stderr) {
|
||||
debug('Final test result', error);
|
||||
|
||||
mailer.sendEndToEndTestResult(topic, stdout ? stdout.toString('utf8') : '', stderr ? stderr.toString('utf8') : '', function () { });
|
||||
return callback(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
debug('e2etest started.');
|
||||
start();
|
50
mailer.js
Normal file
50
mailer.js
Normal file
@ -0,0 +1,50 @@
|
||||
/* jslint node: true */
|
||||
|
||||
'use strict';
|
||||
|
||||
var assert = require('assert'),
|
||||
debug = require('debug')('e2e:mailer'),
|
||||
postmark = require('postmark')(config.postmarkApiKey()),
|
||||
util = require('util');
|
||||
|
||||
exports = module.exports = {
|
||||
sendEndToEndTestResult: sendEndToEndTestResult
|
||||
};
|
||||
|
||||
function send(options, callback) {
|
||||
assert.strictEqual(typeof options, 'object');
|
||||
assert.strictEqual(typeof options.to, 'string');
|
||||
assert.strictEqual(typeof options.subject, 'string');
|
||||
assert.strictEqual(typeof options.text, 'string');
|
||||
|
||||
debug('Sending email to %s with subject "%s".', options.to, options.subject);
|
||||
|
||||
postmark.send({
|
||||
'From': options.from || 'no-reply@cloudron.io',
|
||||
'To': options.to,
|
||||
'Bcc': options.bcc,
|
||||
'Subject': options.subject,
|
||||
'TextBody': options.text,
|
||||
'HtmlBody': options.html,
|
||||
'Tag': 'Important'
|
||||
}, function (error, success) {
|
||||
if (error) {
|
||||
console.error('Unable to send via postmark: ', error);
|
||||
return callback(error);
|
||||
}
|
||||
|
||||
callback();
|
||||
});
|
||||
}
|
||||
|
||||
function sendEndToEndTestResult(topic, stdout, stderr, callback) {
|
||||
debug('Sending e2e test result for %s', topic);
|
||||
|
||||
var mailOptions = {
|
||||
to: 'admin@cloudron.io',
|
||||
subject: util.format('E2E test results for box %s', topic),
|
||||
text: 'stdout\n------\n' + stdout.toString('utf8') + '\n\nstderr\n------\n' + stderr.toString('utf8') + '\n\n'
|
||||
};
|
||||
|
||||
send(mailOptions, callback);
|
||||
}
|
@ -17,6 +17,7 @@
|
||||
"colors": "^1.1.0",
|
||||
"debug": "^2.2.0",
|
||||
"mocha": "^2.2.5",
|
||||
"once": "^1.3.2",
|
||||
"parallel-mocha": "0.0.7",
|
||||
"readline-sync": "^1.2.19",
|
||||
"semver": "^4.3.6",
|
||||
|
103
prepareDeployTarget.sh
Normal file
103
prepareDeployTarget.sh
Normal file
@ -0,0 +1,103 @@
|
||||
#!/bin/bash
|
||||
|
||||
######################################
|
||||
## This script is run on the server ##
|
||||
######################################
|
||||
|
||||
set -eux -o pipefail
|
||||
|
||||
readonly APP_DIR="/home/ubuntu/app"
|
||||
readonly CERTS_DIR="/home/ubuntu/certs"
|
||||
readonly APP_REPO_DIR="/home/ubuntu/app.git"
|
||||
readonly INIT_MASTER_DIR="/tmp/repoMasterSetup"
|
||||
readonly ECOSYSTEM="/home/ubuntu/ecosystem.json"
|
||||
readonly SERVER_NAME="<%= serverName %>"
|
||||
readonly ENV="<%= env %>"
|
||||
|
||||
swap_file="/swap"
|
||||
[[ -f "${swap_file}" ]] && sudo swapoff "${swap_file}"
|
||||
if [[ "${ENV}" == "dev" ]]; then
|
||||
sudo fallocate -l 2048m "${swap_file}" # t1.micro has 1GB RAM
|
||||
elif [[ "${ENV}" == "staging" ]]; then
|
||||
sudo fallocate -l 2048m "${swap_file}" # t1.micro has 1GB RAM
|
||||
else
|
||||
sudo fallocate -l 2048m "${swap_file}" # t1.micro has 1GB RAM
|
||||
fi
|
||||
sudo chmod 600 "${swap_file}"
|
||||
sudo mkswap "${swap_file}"
|
||||
sudo swapon "${swap_file}"
|
||||
if ! grep "${swap_file}" /etc/fstab; then
|
||||
sudo bash -c "echo '${swap_file} none swap sw 0 0' >> /etc/fstab"
|
||||
fi
|
||||
|
||||
# install git
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y git
|
||||
|
||||
# install node v0.12 https://github.com/nodesource/distributions
|
||||
sudo apt-get install -y python-software-properties python g++ make
|
||||
curl -sL https://deb.nodesource.com/setup_0.12 | sudo bash -
|
||||
sudo apt-get install -y nodejs
|
||||
|
||||
# copy over ssh keys
|
||||
sudo mv "/tmp/id_rsa" "${HOME}/.ssh/id_rsa"
|
||||
sudo mv "/tmp/id_rsa.pub" "${HOME}/.ssh/id_rsa.pub"
|
||||
echo -e "Host *.cloudron.me\n StrictHostKeyChecking no" > ${HOME}/.ssh/config
|
||||
|
||||
# install tmpreaper (runs everyday and clean tmp)
|
||||
sudo apt-get install -y tmpreaper
|
||||
sudo sed -e 's/SHOWWARNING=true/# SHOWWARNING=true/' -i /etc/tmpreaper.conf
|
||||
|
||||
# install pm2
|
||||
sudo npm install -g pm2 pm2-run forever
|
||||
sudo rm -rf ~/.npm # .npm will get owned by root after "sudo npm install -g"
|
||||
|
||||
# create deploy directories
|
||||
mkdir -p "${APP_DIR}"
|
||||
|
||||
# prepare git repo in case there is not yet one
|
||||
if [[ ! -d "${APP_REPO_DIR}" ]]; then
|
||||
mkdir -p "${APP_REPO_DIR}"
|
||||
cd "${APP_REPO_DIR}"
|
||||
git init --bare
|
||||
|
||||
# create stub master branch
|
||||
mkdir -p "${INIT_MASTER_DIR}" && cd "${INIT_MASTER_DIR}"
|
||||
git init
|
||||
touch stub
|
||||
git add stub
|
||||
git commit -m "stub"
|
||||
git push ${APP_REPO_DIR} master --force
|
||||
cd && rm -rf "${INIT_MASTER_DIR}"
|
||||
else
|
||||
echo "Git repo already exists at ${APP_REPO_DIR}"
|
||||
fi
|
||||
|
||||
# install post-receive hook
|
||||
cat > "${APP_REPO_DIR}/hooks/post-receive" <<EOF
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
git --work-tree=${APP_DIR} --git-dir=${APP_REPO_DIR} checkout -f
|
||||
cd ${APP_DIR}
|
||||
pm2-run --ecosystem ${ECOSYSTEM} --env <%= env %> --cmd "npm prune && npm install --production"
|
||||
pm2 startOrRestart ${ECOSYSTEM} --env <%= env %>
|
||||
|
||||
# save the new status for restart
|
||||
pm2 save
|
||||
|
||||
EOF
|
||||
chmod +x "${APP_REPO_DIR}/hooks/post-receive"
|
||||
|
||||
|
||||
# create empty ecosystem.json
|
||||
touch "${ECOSYSTEM}"
|
||||
|
||||
# run pm2 as current user ubuntu to avoid pm2 running as root below
|
||||
pm2 status
|
||||
|
||||
# ensure we run the latest version, this might not be the case if this script is ran against a already prepared server
|
||||
pm2 updatePM2
|
||||
|
||||
# setup the init scripts
|
||||
sudo pm2 startup -u ubuntu
|
||||
|
95
shell.js
Normal file
95
shell.js
Normal file
@ -0,0 +1,95 @@
|
||||
'use strict';
|
||||
|
||||
exports = module.exports = {
|
||||
sudo: sudo,
|
||||
exec: exec,
|
||||
system: system
|
||||
};
|
||||
|
||||
var assert = require('assert'),
|
||||
child_process = require('child_process'),
|
||||
debug = require('debug')('e2e:shell.js'),
|
||||
once = require('once'),
|
||||
util = require('util');
|
||||
|
||||
var SUDO = '/usr/bin/sudo';
|
||||
|
||||
function exec(tag, file, args, options, callback) {
|
||||
assert.strictEqual(typeof tag, 'string');
|
||||
assert.strictEqual(typeof file, 'string');
|
||||
assert(util.isArray(args));
|
||||
|
||||
if (typeof options === 'function') {
|
||||
callback = options;
|
||||
options = { };
|
||||
} else {
|
||||
assert.strictEqual(typeof options, 'object');
|
||||
assert.strictEqual(typeof callback, 'function');
|
||||
}
|
||||
|
||||
callback = once(callback); // exit may or may not be called after an 'error'
|
||||
|
||||
debug(tag + ' execFile: %s %s', file, args.join(' '));
|
||||
|
||||
var cp = child_process.spawn(file, args, options);
|
||||
cp.stdout.on('data', function (data) {
|
||||
debug(tag + ' (stdout): %s', data.toString('utf8'));
|
||||
});
|
||||
|
||||
cp.stderr.on('data', function (data) {
|
||||
debug(tag + ' (stderr): %s', data.toString('utf8'));
|
||||
});
|
||||
|
||||
cp.on('exit', function (code, signal) {
|
||||
if (code || signal) debug(tag + ' code: %s, signal: %s', code, signal);
|
||||
|
||||
callback(code === 0 ? null : new Error(util.format('Exited with error %s signal %s', code, signal)));
|
||||
});
|
||||
|
||||
cp.on('error', function (error) {
|
||||
debug(tag + ' code: %s, signal: %s', error.code, error.signal);
|
||||
callback(error);
|
||||
});
|
||||
|
||||
return cp;
|
||||
}
|
||||
|
||||
function system(tag, cmd, options, callback) {
|
||||
assert.strictEqual(typeof tag, 'string');
|
||||
assert.strictEqual(typeof cmd, 'string');
|
||||
|
||||
if (typeof options === 'function') {
|
||||
callback = options;
|
||||
options = { };
|
||||
} else {
|
||||
assert.strictEqual(typeof options, 'object');
|
||||
assert.strictEqual(typeof callback, 'function');
|
||||
}
|
||||
|
||||
debug(tag + ' system: %s %j', cmd, options);
|
||||
|
||||
var cp = child_process.exec(cmd, options, function (error, stdout, stderr) {
|
||||
if (error) debug(tag + ' (error) ' + error);
|
||||
|
||||
callback(error, stdout, stderr);
|
||||
});
|
||||
|
||||
cp.stdout.on('data', function (data) {
|
||||
debug(tag + ' (stdout): %s', data.toString('utf8'));
|
||||
});
|
||||
|
||||
cp.stderr.on('data', function (data) {
|
||||
debug(tag + ' (stderr): %s', data.toString('utf8'));
|
||||
});
|
||||
}
|
||||
|
||||
function sudo(tag, args, callback) {
|
||||
assert.strictEqual(typeof tag, 'string');
|
||||
assert(util.isArray(args));
|
||||
assert.strictEqual(typeof callback, 'function');
|
||||
|
||||
// -S makes sudo read stdin for password
|
||||
var cp = exec(tag, SUDO, [ '-S' ].concat(args), callback);
|
||||
cp.stdin.end();
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user