From 7e57f2ed0b0f032af42c7687d330edb9dad53e08 Mon Sep 17 00:00:00 2001 From: ViViDboarder Date: Tue, 24 Jul 2018 08:38:22 -0700 Subject: [PATCH] Let's get restic in here Switch to restic --- Dockerfile | 27 +++++++++++++++++ Dockerfile.raspbian | 51 -------------------------------- Dockerfile.ubuntu | 51 -------------------------------- Makefile | 10 +++---- Readme.md | 44 ++++++++------------------- backup.sh | 32 +++++++------------- docker-compose.yaml | 4 +-- restore.sh | 21 +++++-------- start.sh | 19 ++++-------- tests/docker-compose-test-s3.yml | 10 +++---- tests/test.sh | 16 ++++++---- verify.sh | 9 ++---- 12 files changed, 90 insertions(+), 204 deletions(-) create mode 100644 Dockerfile delete mode 100644 Dockerfile.raspbian delete mode 100644 Dockerfile.ubuntu diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9b7bf4f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM ubuntu:artful +MAINTAINER ViViDboarder + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + cron \ + restic \ + && apt-get clean \ + && rm -rf /var/apt/lists/* + +VOLUME /root/.cache/restic +VOLUME /backups + +ENV BACKUP_DEST="/backups" +ENV BACKUP_NAME="backup" +ENV PATH_TO_BACKUP="/data" + +# Cron schedules +ENV CRON_SCHEDULE="" +ENV VERIFY_CRON_SCHEDULE="" + +ADD backup.sh / +ADD restore.sh / +ADD start.sh / +ADD verify.sh / + +CMD [ "/start.sh" ] diff --git a/Dockerfile.raspbian b/Dockerfile.raspbian deleted file mode 100644 index 64bbeec..0000000 --- a/Dockerfile.raspbian +++ /dev/null @@ -1,51 +0,0 @@ -FROM resin/rpi-raspbian:jessie -MAINTAINER ViViDboarder - -RUN [ "cross-build-start" ] - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - cron \ - duplicity \ - lftp \ - ncftp \ - openssh-client \ - python-cloudfiles \ - python-gdata \ - python-oauthlib \ - python-paramiko \ - python-pexpect \ - python-pip \ - python-setuptools \ - python-urllib3 \ - rsync \ - tahoe-lafs \ - && pip install -U boto b2 \ - && apt-get remove -y python-pip \ - && apt-get install -y --no-install-recommends \ - python-swiftclient \ - && rm -rf /var/apt/lists/* - -RUN [ "cross-build-end" ] - -VOLUME /root/.cache/duplicity -VOLUME /backups -VOLUME /var/lock/duplicity - -ENV BACKUP_DEST="file:///backups" -ENV BACKUP_NAME="backup" -ENV PATH_TO_BACKUP="/data" -ENV PASSPHRASE="Correct.Horse.Battery.Staple" -ENV FLOCK_WAIT=60 - -# Cron schedules -ENV CRON_SCHEDULE="" -ENV FULL_CRON_SCHEDULE="" -ENV VERIFY_CRON_SCHEDULE="" - -ADD backup.sh / -ADD restore.sh / -ADD start.sh / -ADD verify.sh / - -CMD [ "/start.sh" ] diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu deleted file mode 100644 index 05a030f..0000000 --- a/Dockerfile.ubuntu +++ /dev/null @@ -1,51 +0,0 @@ -FROM ubuntu:xenial -MAINTAINER ViViDboarder - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - software-properties-common python-software-properties \ - && add-apt-repository ppa:duplicity-team/ppa \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - cron \ - duplicity \ - lftp \ - ncftp \ - openssh-client \ - python-cloudfiles \ - python-gdata \ - python-oauthlib \ - python-paramiko \ - python-pexpect \ - python-pip \ - python-setuptools \ - python-swiftclient \ - python-urllib3 \ - rsync \ - tahoe-lafs \ - && pip install -U boto b2 \ - && apt-get autoremove -y python-pip \ - && apt-get clean \ - && rm -rf /var/apt/lists/* - -VOLUME /root/.cache/duplicity -VOLUME /backups -VOLUME /var/lock/duplicity - -ENV BACKUP_DEST="file:///backups" -ENV BACKUP_NAME="backup" -ENV PATH_TO_BACKUP="/data" -ENV PASSPHRASE="Correct.Horse.Battery.Staple" -ENV FLOCK_WAIT=60 - -# Cron schedules -ENV CRON_SCHEDULE="" -ENV FULL_CRON_SCHEDULE="" -ENV VERIFY_CRON_SCHEDULE="" - -ADD backup.sh / -ADD restore.sh / -ADD start.sh / -ADD verify.sh / - -CMD [ "/start.sh" ] diff --git a/Makefile b/Makefile index ee895db..52a8284 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -DOCKER_TAG ?= docker-duplicity-cron +DOCKER_TAG ?= docker-restic-cron .PHONY: default default: build-x86 @@ -8,18 +8,18 @@ test: test-x86 .PHONY: build-x86 build-x86: - docker build -f ./Dockerfile.ubuntu -t $(DOCKER_TAG):ubuntu . + docker build -f ./Dockerfile -t $(DOCKER_TAG) --platform linux . .PHONY: build-arm build-arm: - docker build -f ./Dockerfile.raspbian -t $(DOCKER_TAG):raspbian . + docker build -f ./Dockerfile -t $(DOCKER_TAG) --platform arm . .PHONY: build-all build-all: build-x86 build-arm .PHONY: test-x86 test-x86: build-x86 - cd tests && ./test.sh $(DOCKER_TAG):ubuntu + cd tests && ./test.sh $(DOCKER_TAG) .PHONY: test-arm test-arm: build-arm @@ -41,7 +41,7 @@ test-s3-all: test-s3-x86 test-s3-arm .PHONY: shell-x86 shell-x86: build-x86 - docker run --rm -it $(DOCKER_TAG):ubuntu bash + docker run --rm -it $(DOCKER_TAG) bash .PHONY: shell-arm shell-arm: build-arm diff --git a/Readme.md b/Readme.md index 27e9c25..de59cac 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,6 @@ -# Duplicity Backup +# Restic Backup -[![Build Status](https://travis-ci.org/ViViDboarder/docker-duplicity-cron.svg?branch=master)](https://travis-ci.org/ViViDboarder/docker-duplicity-cron) +[![Build Status](https://travis-ci.org/ViViDboarder/docker-restic-cron.svg?branch=master)](https://travis-ci.org/ViViDboarder/docker-restic-cron) ## Instructions Mount any directories you'd like to back up as a volume and run @@ -8,43 +8,25 @@ Mount any directories you'd like to back up as a volume and run ## Env Variables | Variable | Default | Description | | -------- | ------- | ----------- | -|AWS_ACCESS_KEY_ID| |Required for writing to S3| -|AWS_DEFAULT_REGION| |Required for writing to S3| -|AWS_SECRET_ACCESS_KEY| |Required for writing to S3| -|BACKUP_DEST|file:///backups|Destination to store backups (See [duplicity documenation](http://duplicity.nongnu.org/duplicity.1.html#sect7))| -|BACKUP_NAME|backup|What the name for the backup should be. If using a single store for multiple backups, make sure this is unique| -|CLEANUP_COMMAND| |An optional duplicity command to execute after backups to clean older ones out (eg. "remove-all-but-n-full 2")| +|AWS_ACCESS_KEY_ID| |Required for writing to S3 or Minio| +|AWS_SECRET_ACCESS_KEY| |Required for writing to S3 or Minio| +|B2_ACCOUNT_ID| |Required for writing to B2| +|B2_ACCOUNT_KEY| |Required for writing to B2| +|BACKUP_DEST|/backups|Destination to store backups (See [restic documenation](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html))| +|CLEANUP_COMMAND| |Optional restic arguments for `forget` to execute after backups to clean older ones out (eg. "--prune --keep-last 2"). See [forget](https://restic.readthedocs.io/en/latest/060_forget.html)| |CRON_SCHEDULE| |If you want to periodic incremental backups on a schedule, provide it here. By default we just backup once and exit| -|FLOCK_WAIT|60|Seconds to wait for a lock before skipping a backup| -|FTP_PASSWORD| |Used to provide passwords for some backends. May not work without an attached TTY| |FULL_CRON_SCHEDULE| |If you want to periodic full backups on a schedule, provide it here. This requires an incremental cron schedule too| -|GPG_KEY_ID| |The ID of the key you wish to use. See [Encryption](#encryption) section below| -|OPT_ARGUMENTS| |Any additional arguments to provide to the duplicity backup command| -|PASSPHRASE|Correct.Horse.Battery.Staple|Passphrase to use for GPG| -|PATH_TO_BACKUP|/data|The path to the directory you wish to backup. If you want to backup multiple, see the [tip below](#backing-up-more-than-one-source-directory)| +|OPT_ARGUMENTS| |Any additional arguments to provide to the restic command| +|RESTIC_PASSWORD| |Passphrase to use for encryption| +|PATH_TO_BACKUP|/data|The path to the directory you wish to backup| |RESTORE_ON_EMPTY_START| |Set this to "true" and if the `$PATH_TO_BACKUP` is empty, it will restore the latest backup. This can be used for auto recovery from lost data| |SKIP_ON_START| |Skips backup on start if set to "true"| |VERIFY_CRON_SCHEDULE| |If you want to verify your backups on a schedule, provide it here| -## Encryption -By default Duplicity will use a symettric encryption using just your passphrase. If you wish to use a GPG key, you can add a ro mount to your `~/.gnupg` directory and then provide the `GPG_KEY_ID` as an environment variable. The key will be used to sign and encrypt your files before sending to the backup destination. - -Need to generate a key? Install `gnupg` and run `gnupg --gen-key` - ## Tips -### Missing dependencies? -Please file a ticket! Duplicity supports a ton of backends and I haven't had a chance to validate that all dependencies are present in the image. If something is missing, let me know and I'll add it - -### Getting complains about no terminal for askpass? -Instead of using `FTP_PASSWORD`, add the password to the endpoint url - -### Backing up more than one source directory -Duplicity only accepts one target, however you can refine that selection with `--exclude` and `--include` arguments. The below example shows how this can be used to select multiple backup sources -``` -OPT_ARGUMENTS="--include /home --include /etc --exclude '**'" -PATH_TO_BACKUP="/" -``` +## Hostnames +Hostname is used for identifying what you are backing up. You may want to specify this on your container. ### Backing up from another container Mount all volumes from your existing container with `--volumes-from` and then back up by providing the paths to those volumes. If there are more than one volumes, you'll want to use the above tip for mulitple backup sources diff --git a/backup.sh b/backup.sh index 4473a4d..adf7ab7 100755 --- a/backup.sh +++ b/backup.sh @@ -1,25 +1,15 @@ #! /bin/bash set -e -( - if ! flock -x -w $FLOCK_WAIT 200 ; then - echo 'ERROR: Could not obtain lock. Exiting.' - exit 1 - fi +restic \ + -r $BACKUP_DEST \ + $OPT_ARGUMENTS \ + backup \ + $PATH_TO_BACKUP - duplicity \ - $1 \ - --asynchronous-upload \ - --log-file /root/duplicity.log \ - --name $BACKUP_NAME \ - $OPT_ARGUMENTS \ - $PATH_TO_BACKUP \ - $BACKUP_DEST - - if [ -n "$CLEANUP_COMMAND" ]; then - duplicity $CLEANUP_COMMAND \ - --log-file /root/duplicity.log \ - --name $BACKUP_NAME \ - $BACKUP_DEST - fi -) 200>/var/lock/duplicity/.duplicity.lock +if [ -n "$CLEANUP_COMMAND" ]; then + restic \ + -r $BACKUP_DEST \ + forget \ + $CLEANUP_COMMAND +fi diff --git a/docker-compose.yaml b/docker-compose.yaml index 56b2c1f..e2a43fb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,9 +1,9 @@ version: '2' services: - duplicity: + restic: build: context: . - dockerfile: Dockerfile.ubuntu + dockerfile: Dockerfile volumes: - ./my-backups:/backups - ./my-data:/data:ro diff --git a/restore.sh b/restore.sh index eec941f..b45c4a0 100755 --- a/restore.sh +++ b/restore.sh @@ -1,18 +1,11 @@ #! /bin/bash set -e -( - if ! flock -x -w $FLOCK_WAIT 200 ; then - echo 'ERROR: Could not obtain lock. Exiting.' - exit 1 - fi +restore_snapshot=$1 - duplicity restore \ - --force \ - --log-file /root/duplicity.log \ - --name $BACKUP_NAME \ - $OPT_ARGUMENTS \ - $@ \ - $BACKUP_DEST \ - $PATH_TO_BACKUP -) 200>/var/lock/duplicity/.duplicity.lock +restic \ + -r $BACKUP_DEST \ + $OPT_ARGUMENTS \ + restore \ + $restore_snapshot \ + -t / diff --git a/start.sh b/start.sh index 1480490..47a36ab 100755 --- a/start.sh +++ b/start.sh @@ -11,14 +11,12 @@ if [ "$OPT_ARGUMENTS" == "" ]; then export OPT_ARGUMENTS="$@" fi -# If key id is provied add arg -if [ -e "$GPG_KEY_ID" ]; then - export OPT_ARGUMENTS="$OPT_ARGUMENTS --encrypt-sign-key=\"$GPG_KEY_ID\"" -fi +# Init the repo +restic -r $BACKUP_DEST snapshots || restic -r $BACKUP_DEST init # If set to restore on start, restore if the data volume is empty if [ "$RESTORE_ON_EMPTY_START" == "true" -a -z "$(ls -A $PATH_TO_BACKUP)" ]; then - /restore.sh + /restore.sh latest fi # Unless explicitly skipping, take a backup on startup @@ -39,11 +37,6 @@ if [ -n "$CRON_SCHEDULE" ]; then echo "$CRON_SCHEDULE source /env.sh && /backup.sh 2>> /cron.log" >> /crontab.conf echo "Backups scheduled as $CRON_SCHEDULE" - if [ -n "$FULL_CRON_SCHEDULE" ]; then - echo "$FULL_CRON_SCHEDULE source /env.sh && /backup.sh full 2>> /cron.log" >> /crontab.conf - echo "Full backup scheduled as $VERIFY_CRON_SCHEDULE" - fi - if [ -n "$VERIFY_CRON_SCHEDULE" ]; then echo "$VERIFY_CRON_SCHEDULE source /env.sh && /verify.sh 2>> /cron.log" >> /crontab.conf echo "Verify scheduled as $VERIFY_CRON_SCHEDULE" @@ -52,9 +45,9 @@ if [ -n "$CRON_SCHEDULE" ]; then # Add to crontab crontab /crontab.conf - echo "Starting duplicity cron..." + echo "Starting restic cron..." cron - touch /cron.log /root/duplicity.log - tail -f /cron.log /root/duplicity.log + touch /cron.log + tail -f /cron.log fi diff --git a/tests/docker-compose-test-s3.yml b/tests/docker-compose-test-s3.yml index 24ba128..931a4ff 100644 --- a/tests/docker-compose-test-s3.yml +++ b/tests/docker-compose-test-s3.yml @@ -1,17 +1,17 @@ version: '2' services: - duplicity: + restic: build: context: .. - dockerfile: Dockerfile.${DOCKER_BASE} + dockerfile: Dockerfile entrypoint: "bash" - command: ["/test.sh"] + command: "-c 'sleep 2 && /test.sh'" hostname: itest environment: IN_CONTAINER: 'true' SKIP_ON_START: 'true' - OPT_ARGUMENTS: '--s3-unencrypted-connection' - BACKUP_DEST: s3://minio:9000/duplicity + RESTIC_PASSWORD: Correct.Horse.Battery.Staple + BACKUP_DEST: s3:http://minio:9000/restic AWS_DEFAULT_REGION: us-east-1 AWS_ACCESS_KEY_ID: SUPER_SECRET_ACCESS_KEY AWS_SECRET_ACCESS_KEY: SUPER_SECRET_SECRET_KEY diff --git a/tests/test.sh b/tests/test.sh index af834be..c45a3c8 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -1,4 +1,5 @@ #! /bin/bash +set -e image=$1 @@ -7,6 +8,7 @@ if [ "$IN_CONTAINER" != "true" ] ; then docker run --rm \ -e IN_CONTAINER=true \ -e SKIP_ON_START=true \ + -e RESTIC_PASSWORD="Correct.Horse.Battery.Staple" \ -v "$(pwd)/test.sh:/test.sh" \ $image \ bash -c "/test.sh" @@ -20,6 +22,9 @@ else echo "Create test data..." mkdir -p /data && echo Test > /data/test.txt + echo "Fake a start and init repo" + CRON_SCHEDULE="" /start.sh + echo "Making backup..." /backup.sh @@ -33,7 +38,10 @@ else test -f /data/test.txt && exit 1 || echo "Gone" echo "Restore backup..." - /restore.sh + /restore.sh latest + + echo "Verify restore..." + cat /data/test.txt echo "Verify backup..." /verify.sh @@ -48,11 +56,9 @@ else RESTORE_ON_EMPTY_START=true /start.sh echo "Verify restore happened..." - test -f /data/test.txt + cat /data/test.txt echo "Verify restore with incorrect passphrase fails..." - export PASSPHRASE=Incorrect.Mule.Solar.Paperclip - echo "Fail to restore backup..." - /restore.sh && exit 1 || echo "OK" + RESTIC_PASSWORD=Incorrect.Mule.Solar.Paperclip /restore.sh latest && exit 1 || echo "OK" fi diff --git a/verify.sh b/verify.sh index f11ff35..7dec438 100755 --- a/verify.sh +++ b/verify.sh @@ -1,10 +1,7 @@ #! /bin/bash set -e -duplicity verify \ - --compare-data \ - --log-file /root/duplicity.log \ - --name $BACKUP_NAME \ +restic \ + -r $BACKUP_DEST \ $OPT_ARGUMENTS \ - $BACKUP_DEST \ - $PATH_TO_BACKUP + check