diff --git a/test/selfhost-ec2-create-test.js b/test/selfhost-ec2-create-test.js
index 08390b1..5b1eb66 100644
--- a/test/selfhost-ec2-create-test.js
+++ b/test/selfhost-ec2-create-test.js
@@ -206,7 +206,8 @@ describe('Selfhost EC2 Cloudron creation', function () {
         cloudron.waitForApp(appId);
     });
 
-    it('can check the addons', function () {
+    // Only works so far with upgrades, as the app needs to be restarted for the test to suceed
+    xit('can check the addons', function () {
         cloudron.checkAddons(cloudron.appFqdn(location), owner);
     });