Add initial test for digitalocean with s3 backups

cloudron.js

@@ -633,6 +633,11 @@ Cloudron.prototype.setDnsConfig = function (dnsConfig) {
     common.verifyResponse2xx(res, 'Could not set dns config');
 };
 
+Cloudron.prototype.setBackupConfig = function (backupConfig) {
+    var res = request.post(this._origin + '/api/v1/settings/backup_config').query({ access_token: this._credentials.accessToken }).send(backupConfig).end();
+    common.verifyResponse2xx(res, 'Could not set backup config');
+};
+
 Cloudron.prototype.saveSieveScript = function (owner, callback) {
     var authString = 'AUTHENTICATE "PLAIN" "' + new Buffer('\0' + owner.username + '\0' + owner.password).toString('base64') + '"';
     var data = '';

test/selfhost-digitalocean-filesystem-test.js

@@ -51,7 +51,7 @@ function machine(args, options) {
     }
 }
 
-describe('Selfhost DigitalOcean Cloudron creation', function () {
+describe('Selfhost DigitalOcean with filesystem backend', function () {
     this.timeout(0);
 
     var appStore = new AppStore();

test/selfhost-digitalocean-s3-test.js

@@ -0,0 +1,303 @@
+/*
+ *
+ * This tests a flow for cloudron owner creating a selfhosted cloudron on digitalocean
+ * Owner creates a cloudron, activates it, installs and app and deletes the cloudron eventually
+ * We use a us-east-1 (US standard) backup bucket here
+ *
+ */
+
+'use strict';
+
+var AppStore = require('../appstore.js'),
+    assert = require('assert'),
+    async = require('async'),
+    child_process = require('child_process'),
+    Cloudron = require('../cloudron.js'),
+    common = require('../common.js'),
+    mailer = require('../mailer.js'),
+    rimraf = require('rimraf'),
+    semver = require('semver'),
+    sleep = require('../shell.js').sleep,
+    superagent = require('superagent'),
+    request = require('superagent-sync'),
+    util = require('util');
+
+require('colors');
+
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
+const BOX_VERSION = process.env.BOX_VERSION;
+const DO_SELFHOST_DOMAIN = process.env.DO_SELFHOST_DOMAIN;
+const SSH_KEY = 'id_rsa_e2e_selfhost';
+const DO_TYPE = '1gb';
+const DO_REGION = 'nyc3';
+const DO_TOKEN = process.env.DIGITAL_OCEAN_TOKEN_STAGING;
+const BACKUP_KEY = 'somesecret';
+const AWS_BACKUP_BUCKET = 'selfhost-test-digitalocean';
+const AWS_BACKUP_PREFIX = DO_SELFHOST_DOMAIN;
+const AWS_ACCESS_KEY = process.env.AWS_STAGING_ACCESS_KEY;
+const AWS_ACCESS_SECRET = process.env.AWS_STAGING_SECRET_KEY;
+
+function machine(args, options) {
+    // https://github.com/nodejs/node-v0.x-archive/issues/9265
+    options = options || { };
+    args = util.isArray(args) ? args : args.match(/[^\s"]+|"([^"]+)"/g);
+    args = args.map(function (e) { return e[0] === '"' ? e.slice(1, -1) : e; }); // remove the quotes
+
+    console.log('cloudron machine ' + args.join(' '));
+
+    try {
+        var cp = child_process.spawnSync('cloudron', ['machine'].concat(args), { stdio: [ options.stdin || 'pipe', options.stdout || 'pipe', 'pipe' ], encoding: options.encoding || 'utf8' });
+        return cp;
+    } catch (e) {
+        console.error(e);
+        throw e;
+    }
+}
+
+describe('Selfhost DigitalOcean with S3 backend', function () {
+    this.timeout(0);
+
+    var appStore = new AppStore();
+
+    var owner = common.getOwner();
+    var cloudron, appId, backupInfo, instanceId, restoreInstanceId, migrateInstanceId;
+    var fromVersion, toVersion, nextVersion;
+
+    it('can query versions', function () {
+        var res = request.get(process.env.BOX_VERSIONS_URL).end();
+        common.verifyResponse2xx(res);
+        var boxVersions = Object.keys(common.stripUnreachable(res.body)).sort(semver.rcompare);
+        fromVersion = boxVersions[2]; // we released a new version in before.js
+        toVersion = boxVersions[1];
+        assert.strictEqual(toVersion, BOX_VERSION);
+        nextVersion = boxVersions[0];
+
+        console.log('Will test update from %s to %s and then %s', fromVersion.yellow, toVersion.yellow, nextVersion.yellow);
+    });
+
+    it('can create a cloudron', function () {
+        var params = [
+            '--fqdn ' + DO_SELFHOST_DOMAIN,
+            '--type ' + DO_TYPE,
+            '--token ' + DO_TOKEN,
+            '--region ' + DO_REGION,
+            '--ssh-key ' + SSH_KEY,
+            '--backup-key ' + BACKUP_KEY,
+            '--release ' + toVersion
+        ];
+
+        var out = machine('create digitalocean ' + params.join(' '));
+        console.log(out.stdout, out.stderr);
+
+        if (out.stdout.indexOf('You can now setup your Cloudron at') === -1) {
+            assert(false, 'Creation failed');
+        }
+
+        // Wohooo strings!
+        instanceId = out.stdout.split('\n').filter(function (l) { return l.indexOf(' ID: ') !== -1; })[0].split(':')[1].trim();
+
+        console.log('New instance created with ID', instanceId);
+
+        cloudron = new Cloudron({
+            domain: DO_SELFHOST_DOMAIN,
+            setupToken: null,
+            version: toVersion,
+            ip: null
+        });
+    });
+
+    it('can activate the box', function () {
+        cloudron.activate(owner);
+    });
+
+    it('can login to the box', function () {
+        var token = cloudron.getOauthToken(owner);
+        cloudron.setCredentials(owner.password, token);
+    });
+
+    it('can take a breather', function () {
+        // do can be really slow to come up. the addon containers take their own sweet time (they are "async" with the box startup)
+        // we end up sending email even before the mail container is ready
+        sleep(30);
+    });
+
+    it('can enable email', function () {
+        cloudron.setEmailEnabled(true);
+    });
+
+    it('send mail to cloudron user', function (done) {
+        mailer.sendMailToCloudronUser(owner.username + '@' + DO_SELFHOST_DOMAIN, done);
+    });
+
+    it('did receive mail', function (done) {
+        cloudron.checkMail(owner, done);
+    });
+
+    var location = 'test' + (Math.random() * 10000).toFixed();
+    it('can install app', function () {
+        var manifest = appStore.getManifest(common.TESTAPP_ID, common.TESTAPP_VERSION);
+        appId = cloudron.installApp(location, manifest);
+    });
+
+    it('can populate the addons', function () {
+        cloudron.populateAddons(cloudron.appFqdn(location));
+    });
+
+    it('can check the addons', function () {
+        cloudron.checkAddons(cloudron.appFqdn(location), owner);
+    });
+
+    it('can set S3 backup config', function () {
+        var backupConfig = {
+            accessKeyId: AWS_ACCESS_KEY,
+            secretAccessKey: AWS_ACCESS_SECRET,
+            bucket: AWS_BACKUP_BUCKET,
+            prefix: AWS_BACKUP_PREFIX
+        };
+
+        cloudron.setBackupConfig(backupConfig);
+    });
+
+    it('can update the box', function () {
+        cloudron.checkForUpdates();
+
+        var params = [
+            DO_SELFHOST_DOMAIN,
+            '--yes',
+            '--ssh-key ' + SSH_KEY,
+            '--username ' + owner.username,
+            '--password ' + owner.password
+        ];
+
+        var out = machine('update ' + params.join(' '));
+        console.log(out.stdout, out.stderr);
+
+        if (out.stdout.indexOf('You can now use your Cloudron at') === -1) {
+            assert(false, 'Update failed');
+        }
+    });
+
+    it('wait for app to be ready', function () {
+        cloudron.waitForApp(appId);
+    });
+
+    it('can configure app', function () {
+        location = location + 'x';
+        cloudron.configureApp(appId, location);
+    });
+
+    it('can check the addons', function () {
+        cloudron.checkAddons(cloudron.appFqdn(location), owner);
+    });
+
+    it('can reboot the cloudron', function () {
+        cloudron.reboot();
+    });
+
+    it('runs the app', function () {
+        cloudron.waitForApp(appId);
+    });
+
+    it('can check the addons', function () {
+        cloudron.checkAddons(cloudron.appFqdn(location), owner);
+    });
+
+    it('can check mail', function (done) {
+        cloudron.checkMail(owner, done);
+    });
+
+    it('can backup the box', function () {
+        backupInfo = cloudron.backup();
+        assert.strictEqual(backupInfo.dependsOn.length, 1);
+    });
+
+    it('can list backups', function () {
+        var out = machine('backup list ' + DO_SELFHOST_DOMAIN);
+        console.log(out.stdout, out.stderr);
+
+        var lastBackupId = out.stdout.split('\n').filter(function (l) { return l.indexOf('backup_') === 0; }).map(function (l) { return l.split(' ')[0].trim(); }).pop();
+
+        console.log('Last backup id:', lastBackupId);
+
+        assert.equal(lastBackupId, backupInfo.id);
+    });
+
+    it('can restore the box', function () {
+        var params = [
+            '--fqdn ' + DO_SELFHOST_DOMAIN,
+            // THOSE SHOULD BE STASHED IN THE CONFIG
+            // '--type ' + DO_TYPE,
+            // '--token ' + DO_TOKEN,
+            // '--region ' + DO_REGION,
+            // '--ssh-key ' + SSH_KEY,
+            // '--backup-key ' + BACKUP_KEY,
+            '--backup ' + backupInfo.id
+        ];
+
+        var out = machine('restore digitalocean ' + params.join(' '));
+        console.log(out.stdout, out.stderr);
+
+        if (out.stdout.indexOf('You can now use your Cloudron at') === -1) {
+            assert(false, 'Restore failed');
+        }
+
+        restoreInstanceId = out.stdout.split('\n').filter(function (l) { return l.indexOf(' ID: ') !== -1; })[0].split(':')[1].trim();
+
+        console.log('New instance created with ID', restoreInstanceId);
+    });
+
+    it('runs the app', function () {
+        cloudron.waitForApp(appId);
+    });
+
+    // Only works so far with upgrades, as the app needs to be restarted for the test to suceed
+    xit('can check the addons', function () {
+        cloudron.checkAddons(cloudron.appFqdn(location), owner);
+    });
+
+    it('can migrate cloudron', function () {
+        var params = [
+            '--fqdn ' + DO_SELFHOST_DOMAIN,
+            // THOSE SHOULD BE STASHED IN THE CONFIG
+            // '--token ' + DO_TOKEN,
+            // '--ssh-key ' + SSH_KEY
+        ];
+
+        var out = machine('migrate digitalocean ' + params.join(' '));
+        console.log(out.stdout, out.stderr);
+
+        if (out.stdout.indexOf('You can now use your Cloudron at') === -1) {
+            assert(false, 'Migrate failed');
+        }
+
+        migrateInstanceId = out.stdout.split('\n').filter(function (l) { return l.indexOf(' ID: ') !== -1; })[0].split(':')[1].trim();
+
+        console.log('New instance created with ID', restoreInstanceId);
+    });
+
+    it('runs the app', function () {
+        cloudron.waitForApp(appId);
+    });
+
+    it('can uninstall app', function () {
+        cloudron.uninstallApp(appId);
+    });
+
+    it('can delete the cloudron', function (done) {
+        console.log('Cleanup DO instances', instanceId, restoreInstanceId, migrateInstanceId);
+
+        // we ignore errors here
+        function deleteDroplet(id, callback) {
+            if (!id) return callback();
+
+            superagent.del('https://api.digitalocean.com/v2/droplets/' + id).set('Authorization', 'Bearer ' + DO_TOKEN).end(function (error, result) {
+                if (error) console.error(error.message);
+                if (result.statusCode !== 204) console.error('Failed to cleanup old droplet. %s %j', result.statusCode, result.body);
+
+                callback();
+            });
+        }
+
+        async.each([ instanceId, restoreInstanceId, migrateInstanceId ], deleteDroplet, done);
+    });
+});