From 9671c1461d05b4d6ad21354f93353034ad454b35 Mon Sep 17 00:00:00 2001 From: ViViDboarder Date: Fri, 10 Aug 2018 11:20:47 -0700 Subject: [PATCH 1/4] Update test to be more testy --- tests/test.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test.sh b/tests/test.sh index af834be..84546dc 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -1,4 +1,5 @@ #! /bin/bash +set -e image=$1 @@ -35,6 +36,10 @@ else echo "Restore backup..." /restore.sh + echo "Verify restore..." + test -f /data/test.txt + cat /data/test.txt + echo "Verify backup..." /verify.sh @@ -49,10 +54,9 @@ else echo "Verify restore happened..." test -f /data/test.txt + cat /data/test.txt echo "Verify restore with incorrect passphrase fails..." - export PASSPHRASE=Incorrect.Mule.Solar.Paperclip - echo "Fail to restore backup..." - /restore.sh && exit 1 || echo "OK" + PASSPHRASE=Incorrect.Mule.Solar.Paperclip /restore.sh && exit 1 || echo "OK" fi From 7b4e31c00d133d1034ff1f31cc765a52130f54ca Mon Sep 17 00:00:00 2001 From: Ian Date: Wed, 15 Aug 2018 18:44:00 -0700 Subject: [PATCH 2/4] ARM: Move setuptools to pip from apt Try to fix build error on ARM --- Dockerfile.raspbian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.raspbian b/Dockerfile.raspbian index 64bbeec..c94ac05 100644 --- a/Dockerfile.raspbian +++ b/Dockerfile.raspbian @@ -16,10 +16,10 @@ RUN apt-get update \ python-paramiko \ python-pexpect \ python-pip \ - python-setuptools \ python-urllib3 \ rsync \ tahoe-lafs \ + && pip install -U setuptools \ && pip install -U boto b2 \ && apt-get remove -y python-pip \ && apt-get install -y --no-install-recommends \ From b2d95cc7755af3f84a4ee50b1af41795ea67acf0 Mon Sep 17 00:00:00 2001 From: ViViDboarder Date: Thu, 27 Sep 2018 17:32:55 -0700 Subject: [PATCH 3/4] Add support for pre/post scripts Allows adding scripts to a directory to be executed before backups and after restorating. This can allow backing up a mysql dump or something. Using these could (and probably does) break verify checks Fixes #8 --- Dockerfile.raspbian | 6 ++++ Dockerfile.ubuntu | 6 ++++ Makefile | 2 ++ backup.sh | 15 ++++++++ restore.sh | 15 ++++++++ tests/scripts/post-backup.sh | 4 +++ tests/scripts/post-restore.sh | 3 ++ tests/scripts/pre-backup.sh | 7 ++++ tests/scripts/pre-restore.sh | 3 ++ tests/test-pre-scripts.sh | 68 +++++++++++++++++++++++++++++++++++ 10 files changed, 129 insertions(+) create mode 100755 tests/scripts/post-backup.sh create mode 100755 tests/scripts/post-restore.sh create mode 100755 tests/scripts/pre-backup.sh create mode 100755 tests/scripts/pre-restore.sh create mode 100755 tests/test-pre-scripts.sh diff --git a/Dockerfile.raspbian b/Dockerfile.raspbian index c94ac05..9e039be 100644 --- a/Dockerfile.raspbian +++ b/Dockerfile.raspbian @@ -43,6 +43,12 @@ ENV CRON_SCHEDULE="" ENV FULL_CRON_SCHEDULE="" ENV VERIFY_CRON_SCHEDULE="" +# Create script dirs +RUN mkdir -p /scripts/backup/before +RUN mkdir -p /scripts/backup/after +RUN mkdir -p /scripts/restore/before +RUN mkdir -p /scripts/restore/after + ADD backup.sh / ADD restore.sh / ADD start.sh / diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 05a030f..dce74a7 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -43,6 +43,12 @@ ENV CRON_SCHEDULE="" ENV FULL_CRON_SCHEDULE="" ENV VERIFY_CRON_SCHEDULE="" +# Create script dirs +RUN mkdir -p /scripts/backup/before +RUN mkdir -p /scripts/backup/after +RUN mkdir -p /scripts/restore/before +RUN mkdir -p /scripts/restore/after + ADD backup.sh / ADD restore.sh / ADD start.sh / diff --git a/Makefile b/Makefile index ee895db..72d292e 100644 --- a/Makefile +++ b/Makefile @@ -20,10 +20,12 @@ build-all: build-x86 build-arm .PHONY: test-x86 test-x86: build-x86 cd tests && ./test.sh $(DOCKER_TAG):ubuntu + cd tests && ./test-pre-scripts.sh $(DOCKER_TAG):ubuntu .PHONY: test-arm test-arm: build-arm cd tests && .test.sh $(DOCKER_TAG):raspbian + cd tests && ./test-pre-scripts.sh $(DOCKER_TAG):raspbian .PHONY: test-all test-all: test-x86 test-arm diff --git a/backup.sh b/backup.sh index 4473a4d..9115891 100755 --- a/backup.sh +++ b/backup.sh @@ -7,6 +7,13 @@ set -e exit 1 fi + # Run pre-backup scripts + for f in /scripts/backup/before/*; do + if [ -f $f -a -x $f ]; then + bash $f + fi + done + duplicity \ $1 \ --asynchronous-upload \ @@ -22,4 +29,12 @@ set -e --name $BACKUP_NAME \ $BACKUP_DEST fi + + # Run post-backup scripts + for f in /scripts/backup/after/*; do + if [ -f $f -a -x $f ]; then + bash $f + fi + done + ) 200>/var/lock/duplicity/.duplicity.lock diff --git a/restore.sh b/restore.sh index eec941f..fc2dec7 100755 --- a/restore.sh +++ b/restore.sh @@ -7,6 +7,13 @@ set -e exit 1 fi + # Run pre-restore scripts + for f in /scripts/restore/before/*; do + if [ -f $f -a -x $f ]; then + bash $f + fi + done + duplicity restore \ --force \ --log-file /root/duplicity.log \ @@ -15,4 +22,12 @@ set -e $@ \ $BACKUP_DEST \ $PATH_TO_BACKUP + + # Run post-restore scripts + for f in /scripts/restore/after/*; do + if [ -f $f -a -x $f ]; then + bash $f + fi + done + ) 200>/var/lock/duplicity/.duplicity.lock diff --git a/tests/scripts/post-backup.sh b/tests/scripts/post-backup.sh new file mode 100755 index 0000000..827f073 --- /dev/null +++ b/tests/scripts/post-backup.sh @@ -0,0 +1,4 @@ +set -e + +# Clear dump +rm /data/dump.txt diff --git a/tests/scripts/post-restore.sh b/tests/scripts/post-restore.sh new file mode 100755 index 0000000..2c7f8c7 --- /dev/null +++ b/tests/scripts/post-restore.sh @@ -0,0 +1,3 @@ +set -e + +cat /data/dump.txt > /mydb.txt diff --git a/tests/scripts/pre-backup.sh b/tests/scripts/pre-backup.sh new file mode 100755 index 0000000..3b86ba7 --- /dev/null +++ b/tests/scripts/pre-backup.sh @@ -0,0 +1,7 @@ +set -e + +# Pretend we have some database +echo "insert foo into bar;" > /mydb.txt + +# Let's dump that db +cat /mydb.txt > /data/dump.txt diff --git a/tests/scripts/pre-restore.sh b/tests/scripts/pre-restore.sh new file mode 100755 index 0000000..ea3e3e4 --- /dev/null +++ b/tests/scripts/pre-restore.sh @@ -0,0 +1,3 @@ +set -e + +# Don't really need to do anything here diff --git a/tests/test-pre-scripts.sh b/tests/test-pre-scripts.sh new file mode 100755 index 0000000..91b00ec --- /dev/null +++ b/tests/test-pre-scripts.sh @@ -0,0 +1,68 @@ +#! /bin/bash +set -e + +image=$1 + +if [ "$IN_CONTAINER" != "true" ] ; then + # Run the test script within the container + docker run --rm \ + -e IN_CONTAINER=true \ + -e SKIP_ON_START=true \ + -v "$(pwd)/test-pre-scripts.sh:/test.sh" \ + -v "$(pwd)/scripts/pre-backup.sh:/scripts/backup/before/1.sh" \ + -v "$(pwd)/scripts/post-backup.sh:/scripts/backup/after/1.sh" \ + -v "$(pwd)/scripts/pre-restore.sh:/scripts/restore/before/1.sh" \ + -v "$(pwd)/scripts/post-restore.sh:/scripts/restore/after/1.sh" \ + $image \ + bash -c "/test.sh" +else + echo "Performing backup tests" + + echo "Verify cron and crontab exist" + type cron + type crontab + + echo "Create test data..." + mkdir -p /data && echo Test > /data/test.txt + + echo "Making backup..." + /backup.sh + + echo "Delete test data..." + rm -fr /data/* + rm -fr /mydb.txt + + echo "Verify deleted..." + test -f /data/test.txt && exit 1 || echo "Gone" + test -f /mydb.txt && exit 1 || echo "Gone" + + echo "Restore backup..." + /restore.sh + + echo "Verify pre-post script backups..." + test -f /mydb.txt + cat /mydb.txt + + echo "Delete test data again..." + rm -fr /data/* + rm -fr /mydb.txt + + echo "Verify deleted..." + test -f /data/test.txt && exit 1 || echo "Gone" + test -f /mydb.txt && exit 1 || echo "Gone" + + echo "Simulate a restart with RESTORE_ON_EMPTY_START..." + RESTORE_ON_EMPTY_START=true /start.sh + + echo "Verify restore happened..." + test -f /data/test.txt + cat /data/test.txt + + echo "Verify pre-post script backups..." + test -f /mydb.txt + cat /mydb.txt + + echo "Verify restore with incorrect passphrase fails..." + echo "Fail to restore backup..." + PASSPHRASE=Incorrect.Mule.Solar.Paperclip /restore.sh && exit 1 || echo "OK" +fi From aad79a458a6fa2c719bca4c880f9342a07ba0204 Mon Sep 17 00:00:00 2001 From: ViViDboarder Date: Mon, 28 Jan 2019 09:53:41 -0800 Subject: [PATCH 4/4] Refactor pre/post script tests --- tests/scripts/post-backup.sh | 4 -- tests/scripts/post-restore.sh | 3 -- tests/scripts/pre-backup.sh | 7 ---- tests/test-pre-scripts.sh | 41 ++++++++++--------- .../backup/after/post-backup.sh | 6 +++ .../backup/before/pre-backup.sh | 6 +++ tests/test-pre-scripts/create-test-data.sql | 7 ++++ .../restore/after/post-restore.sh | 6 +++ .../restore/before}/pre-restore.sh | 0 9 files changed, 47 insertions(+), 33 deletions(-) delete mode 100755 tests/scripts/post-backup.sh delete mode 100755 tests/scripts/post-restore.sh delete mode 100755 tests/scripts/pre-backup.sh create mode 100755 tests/test-pre-scripts/backup/after/post-backup.sh create mode 100755 tests/test-pre-scripts/backup/before/pre-backup.sh create mode 100644 tests/test-pre-scripts/create-test-data.sql create mode 100755 tests/test-pre-scripts/restore/after/post-restore.sh rename tests/{scripts => test-pre-scripts/restore/before}/pre-restore.sh (100%) diff --git a/tests/scripts/post-backup.sh b/tests/scripts/post-backup.sh deleted file mode 100755 index 827f073..0000000 --- a/tests/scripts/post-backup.sh +++ /dev/null @@ -1,4 +0,0 @@ -set -e - -# Clear dump -rm /data/dump.txt diff --git a/tests/scripts/post-restore.sh b/tests/scripts/post-restore.sh deleted file mode 100755 index 2c7f8c7..0000000 --- a/tests/scripts/post-restore.sh +++ /dev/null @@ -1,3 +0,0 @@ -set -e - -cat /data/dump.txt > /mydb.txt diff --git a/tests/scripts/pre-backup.sh b/tests/scripts/pre-backup.sh deleted file mode 100755 index 3b86ba7..0000000 --- a/tests/scripts/pre-backup.sh +++ /dev/null @@ -1,7 +0,0 @@ -set -e - -# Pretend we have some database -echo "insert foo into bar;" > /mydb.txt - -# Let's dump that db -cat /mydb.txt > /data/dump.txt diff --git a/tests/test-pre-scripts.sh b/tests/test-pre-scripts.sh index 91b00ec..8482406 100755 --- a/tests/test-pre-scripts.sh +++ b/tests/test-pre-scripts.sh @@ -9,10 +9,7 @@ if [ "$IN_CONTAINER" != "true" ] ; then -e IN_CONTAINER=true \ -e SKIP_ON_START=true \ -v "$(pwd)/test-pre-scripts.sh:/test.sh" \ - -v "$(pwd)/scripts/pre-backup.sh:/scripts/backup/before/1.sh" \ - -v "$(pwd)/scripts/post-backup.sh:/scripts/backup/after/1.sh" \ - -v "$(pwd)/scripts/pre-restore.sh:/scripts/restore/before/1.sh" \ - -v "$(pwd)/scripts/post-restore.sh:/scripts/restore/after/1.sh" \ + -v "$(pwd)/test-pre-scripts:/scripts" \ $image \ bash -c "/test.sh" else @@ -22,45 +19,51 @@ else type cron type crontab + echo "Install sqlite3" + apt-get update + apt-get install -y --no-install-recommends sqlite3 + echo "Create test data..." - mkdir -p /data && echo Test > /data/test.txt + mkdir -p /data + touch /data/test_database.db + sqlite3 /data/test_database.db < /scripts/create-test-data.sql echo "Making backup..." /backup.sh + echo "Verify intermediary file is gone" + test -f /data/test_database.db.bak && exit 1 || echo "Gone" + echo "Delete test data..." rm -fr /data/* - rm -fr /mydb.txt echo "Verify deleted..." - test -f /data/test.txt && exit 1 || echo "Gone" - test -f /mydb.txt && exit 1 || echo "Gone" + test -f /data/test_database.db && exit 1 || echo "Gone" echo "Restore backup..." /restore.sh - echo "Verify pre-post script backups..." - test -f /mydb.txt - cat /mydb.txt + echo "Verify restored files exist..." + test -f /data/test_database.db + test -f /data/test_database.db.bak && exit 1 || echo "Gone" + sqlite3 /data/test_database.db "select data from test_table where id = 1" echo "Delete test data again..." rm -fr /data/* - rm -fr /mydb.txt echo "Verify deleted..." - test -f /data/test.txt && exit 1 || echo "Gone" - test -f /mydb.txt && exit 1 || echo "Gone" + test -f /data/test_database.db && exit 1 || echo "Gone" echo "Simulate a restart with RESTORE_ON_EMPTY_START..." RESTORE_ON_EMPTY_START=true /start.sh echo "Verify restore happened..." - test -f /data/test.txt - cat /data/test.txt + test -f /data/test_database.db + test -f /data/test_database.db.bak && exit 1 || echo "Gone" + sqlite3 /data/test_database.db "select data from test_table where id = 1" - echo "Verify pre-post script backups..." - test -f /mydb.txt - cat /mydb.txt + echo "Delete test data..." + rm -fr /data/* echo "Verify restore with incorrect passphrase fails..." echo "Fail to restore backup..." diff --git a/tests/test-pre-scripts/backup/after/post-backup.sh b/tests/test-pre-scripts/backup/after/post-backup.sh new file mode 100755 index 0000000..a606858 --- /dev/null +++ b/tests/test-pre-scripts/backup/after/post-backup.sh @@ -0,0 +1,6 @@ +set -e + +cd /data + +# Remove backed up copy +rm test_database.db.bak diff --git a/tests/test-pre-scripts/backup/before/pre-backup.sh b/tests/test-pre-scripts/backup/before/pre-backup.sh new file mode 100755 index 0000000..b0da835 --- /dev/null +++ b/tests/test-pre-scripts/backup/before/pre-backup.sh @@ -0,0 +1,6 @@ +set -e + +cd /data + +# Dump the SQLite database +sqlite3 test_database.db ".backup test_database.db.bak" diff --git a/tests/test-pre-scripts/create-test-data.sql b/tests/test-pre-scripts/create-test-data.sql new file mode 100644 index 0000000..0d63db8 --- /dev/null +++ b/tests/test-pre-scripts/create-test-data.sql @@ -0,0 +1,7 @@ +CREATE TABLE test_table ( + id integer PRIMARY KEY, + data text NOT NULL +); + +INSERT INTO test_table (data) +VALUES ("Test row"); diff --git a/tests/test-pre-scripts/restore/after/post-restore.sh b/tests/test-pre-scripts/restore/after/post-restore.sh new file mode 100755 index 0000000..7ede126 --- /dev/null +++ b/tests/test-pre-scripts/restore/after/post-restore.sh @@ -0,0 +1,6 @@ +set -e + +cd /data + +# Restore the backedup database +mv test_database.db.bak test_database.db diff --git a/tests/scripts/pre-restore.sh b/tests/test-pre-scripts/restore/before/pre-restore.sh similarity index 100% rename from tests/scripts/pre-restore.sh rename to tests/test-pre-scripts/restore/before/pre-restore.sh