Merge remote-tracking branch 'duplicity/health-check' into health-check

Dockerfile

@@ -20,9 +20,13 @@ ENV PATH_TO_BACKUP="/data"
 ENV CRON_SCHEDULE=""
 ENV VERIFY_CRON_SCHEDULE=""
 
-ADD backup.sh /
-ADD restore.sh /
-ADD start.sh /
-ADD verify.sh /
+COPY backup.sh /
+COPY restore.sh /
+COPY start.sh /
+COPY verify.sh /
+COPY healthcheck.sh /
+COPY cron-exec.sh /
+
+HEALTHCHECK CMD /healthcheck.sh
 
 CMD [ "/start.sh" ]

cron-exec.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+ENV=/env
+LOG=/cron.log
+HEALTH_FILE=/unhealthy
+
+test -f $ENV || echo NO_ENV=true > $ENV
+
+# Execute command and write output to log
+env `cat $ENV | xargs` $@ 2>> $LOG && rm -f $HEALTH_FILE || { touch $HEALTH_FILE; exit 1; }

healthcheck.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+HEALTH_FILE=/unhealthy
+
+test -f $HEALTH_FILE || exit 0 && exit 1

start.sh

@@ -16,29 +16,29 @@ restic -r $BACKUP_DEST snapshots || restic -r $BACKUP_DEST init
 
 # If set to restore on start, restore if the data volume is empty
 if [ "$RESTORE_ON_EMPTY_START" == "true" -a -z "$(ls -A $PATH_TO_BACKUP)" ]; then
-    /restore.sh latest
+    /cron-exec.sh /restore.sh latest
 fi
 
 # Unless explicitly skipping, take a backup on startup
 if [ "$SKIP_ON_START" != "true" ]; then
-    /backup.sh
+    /cron-exec.sh /backup.sh
 fi
 
 if [ -n "$CRON_SCHEDULE" ]; then
     # Export the environment to a file so it can be loaded from cron
-    env | sed 's/^\(.*\)=\(.*\)$/export \1="\2"/g' > /env.sh
+    env > /env
     # Remove some vars we don't want to keep
-    sed -i '/\(HOSTNAME\|affinity\|SHLVL\|PWD\)/d' /env.sh
+    sed -i '/\(HOSTNAME\|affinity\|SHLVL\|PWD\)/d' /env
 
     # Use bash for cron
     echo "SHELL=/bin/bash" > /crontab.conf
 
     # Schedule the backups
-    echo "$CRON_SCHEDULE source /env.sh && /backup.sh 2>> /cron.log" >> /crontab.conf
+    echo "$CRON_SCHEDULE /cron-exec.sh /backup.sh" >> /crontab.conf
     echo "Backups scheduled as $CRON_SCHEDULE"
 
     if [ -n "$VERIFY_CRON_SCHEDULE" ]; then
-        echo "$VERIFY_CRON_SCHEDULE source /env.sh && /verify.sh 2>> /cron.log" >> /crontab.conf
+        echo "$VERIFY_CRON_SCHEDULE /cron-exec.sh /verify.sh" >> /crontab.conf
         echo "Verify scheduled as $VERIFY_CRON_SCHEDULE"
     fi
 

tests/test.sh

@@ -26,10 +26,10 @@ else
     CRON_SCHEDULE="" /start.sh
 
     echo "Making backup..."
-    /backup.sh
+    /cron-exec.sh /backup.sh || { cat /cron.log && exit 1; }
 
     echo "Verify backup..."
-    /verify.sh
+    /cron-exec.sh /verify.sh || { cat /cron.log && exit 1; }
 
     echo "Delete test data..."
     rm -fr /data/*
@@ -38,10 +38,8 @@ else
     test -f /data/test.txt && exit 1 || echo "Gone"
 
     echo "Restore backup..."
-    /restore.sh latest
-
-    echo "Verify restore..."
-    cat /data/test.txt
+    /cron-exec.sh /restore.sh latest || { cat /cron.log && exit 1; }
+    /healthcheck.sh
 
     echo "Verify restore..."
     test -f /data/test.txt
@@ -57,7 +55,8 @@ else
     test -f /data/test.txt && exit 1 || echo "Gone"
 
     echo "Simulate a restart with RESTORE_ON_EMPTY_START..."
-    RESTORE_ON_EMPTY_START=true /start.sh
+    RESTORE_ON_EMPTY_START=true /start.sh || { cat /cron.log && exit 1; }
+    /healthcheck.sh || { echo "Failed healthcheck"; cat /cron.log; exit 1; }
 
     echo "Verify restore happened..."
     test -f /data/test.txt
@@ -65,5 +64,8 @@ else
 
     echo "Verify restore with incorrect passphrase fails..."
     echo "Fail to restore backup..."
-    RESTIC_PASSWORD=Incorrect.Mule.Solar.Paperclip /restore.sh latest && exit 1 || echo "OK"
+    RESTIC_PASSWORD=Incorrect.Mule.Solar.Paperclip /cron-exec.sh /restore.sh latest && exit 1 || echo "OK"
+
+    echo "Verify failed healthcheck"
+    /healthcheck.sh && exit 1 || echo "OK"
 fi