Let's get restic in here

Switch to restic
2018-07-24 08:38:22 -07:00 · 2018-07-24 08:38:22 -07:00 · 7e57f2ed0b
parent 947dc939f2
commit 7e57f2ed0b
12 changed files with 90 additions and 204 deletions
--- a/27
+++ b/27
@ -0,0 +1,27 @@
+FROM ubuntu:artful
+MAINTAINER ViViDboarder <vividboarder@gmail.com>
+
+RUN apt-get update \
+        && apt-get install -y --no-install-recommends \
+            cron \
+            restic \
+        && apt-get clean \
+        && rm -rf /var/apt/lists/*
+
+VOLUME /root/.cache/restic
+VOLUME /backups
+
+ENV BACKUP_DEST="/backups"
+ENV BACKUP_NAME="backup"
+ENV PATH_TO_BACKUP="/data"
+
+# Cron schedules
+ENV CRON_SCHEDULE=""
+ENV VERIFY_CRON_SCHEDULE=""
+
+ADD backup.sh /
+ADD restore.sh /
+ADD start.sh /
+ADD verify.sh /
+
+CMD [ "/start.sh" ]
--- a/Dockerfile.raspbian
+++ b/Dockerfile.raspbian
@ -1,51 +0,0 @@
-FROM resin/rpi-raspbian:jessie
-MAINTAINER ViViDboarder <vividboarder@gmail.com>
-
-RUN [ "cross-build-start" ]
-
-RUN apt-get update \
-        && apt-get install -y --no-install-recommends \
-            cron \
-            duplicity \
-            lftp \
-            ncftp \
-            openssh-client \
-            python-cloudfiles \
-            python-gdata \
-            python-oauthlib \
-            python-paramiko \
-            python-pexpect \
-            python-pip \
-            python-setuptools \
-            python-urllib3 \
-            rsync \
-            tahoe-lafs \
-        && pip install -U boto b2 \
-        && apt-get remove -y python-pip \
-        && apt-get install -y --no-install-recommends \
-            python-swiftclient \
-        && rm -rf /var/apt/lists/*
-
-RUN [ "cross-build-end" ]
-
-VOLUME /root/.cache/duplicity
-VOLUME /backups
-VOLUME /var/lock/duplicity
-
-ENV BACKUP_DEST="file:///backups"
-ENV BACKUP_NAME="backup"
-ENV PATH_TO_BACKUP="/data"
-ENV PASSPHRASE="Correct.Horse.Battery.Staple"
-ENV FLOCK_WAIT=60
-
-# Cron schedules
-ENV CRON_SCHEDULE=""
-ENV FULL_CRON_SCHEDULE=""
-ENV VERIFY_CRON_SCHEDULE=""
-
-ADD backup.sh /
-ADD restore.sh /
-ADD start.sh /
-ADD verify.sh /
-
-CMD [ "/start.sh" ]
--- a/Dockerfile.ubuntu
+++ b/Dockerfile.ubuntu
@ -1,51 +0,0 @@
-FROM ubuntu:xenial
-MAINTAINER ViViDboarder <vividboarder@gmail.com>
-
-RUN apt-get update \
-        && apt-get install -y --no-install-recommends \
-            software-properties-common python-software-properties \
-        && add-apt-repository ppa:duplicity-team/ppa \
-        && apt-get update \
-        && apt-get install -y --no-install-recommends \
-            cron \
-            duplicity \
-            lftp \
-            ncftp \
-            openssh-client \
-            python-cloudfiles \
-            python-gdata \
-            python-oauthlib \
-            python-paramiko \
-            python-pexpect \
-            python-pip \
-            python-setuptools \
-            python-swiftclient \
-            python-urllib3 \
-            rsync \
-            tahoe-lafs \
-        && pip install -U boto b2 \
-        && apt-get autoremove -y python-pip \
-        && apt-get clean \
-        && rm -rf /var/apt/lists/*
-
-VOLUME /root/.cache/duplicity
-VOLUME /backups
-VOLUME /var/lock/duplicity
-
-ENV BACKUP_DEST="file:///backups"
-ENV BACKUP_NAME="backup"
-ENV PATH_TO_BACKUP="/data"
-ENV PASSPHRASE="Correct.Horse.Battery.Staple"
-ENV FLOCK_WAIT=60
-
-# Cron schedules
-ENV CRON_SCHEDULE=""
-ENV FULL_CRON_SCHEDULE=""
-ENV VERIFY_CRON_SCHEDULE=""
-
-ADD backup.sh /
-ADD restore.sh /
-ADD start.sh /
-ADD verify.sh /
-
-CMD [ "/start.sh" ]
--- a/10
+++ b/10
@ -1,4 +1,4 @@
-DOCKER_TAG ?= docker-duplicity-cron
+DOCKER_TAG ?= docker-restic-cron

 .PHONY: default
 default: build-x86
@ -8,18 +8,18 @@ test: test-x86

 .PHONY: build-x86
 build-x86:
-	docker build -f ./Dockerfile.ubuntu -t $(DOCKER_TAG):ubuntu .
+	docker build -f ./Dockerfile -t $(DOCKER_TAG) --platform linux .

 .PHONY: build-arm
 build-arm:
-	docker build -f ./Dockerfile.raspbian -t $(DOCKER_TAG):raspbian .
+	docker build -f ./Dockerfile -t $(DOCKER_TAG) --platform arm .

 .PHONY: build-all
 build-all: build-x86 build-arm

 .PHONY: test-x86
 test-x86: build-x86
-	cd tests && ./test.sh $(DOCKER_TAG):ubuntu
+	cd tests && ./test.sh $(DOCKER_TAG)

 .PHONY: test-arm
 test-arm: build-arm
@ -41,7 +41,7 @@ test-s3-all: test-s3-x86 test-s3-arm

 .PHONY: shell-x86
 shell-x86: build-x86
-	docker run --rm -it $(DOCKER_TAG):ubuntu bash
+	docker run --rm -it $(DOCKER_TAG) bash

 .PHONY: shell-arm
 shell-arm: build-arm
--- a/Readme.md
+++ b/Readme.md
@ -1,6 +1,6 @@
-# Duplicity Backup
+# Restic Backup

-[![Build Status](https://travis-ci.org/ViViDboarder/docker-duplicity-cron.svg?branch=master)](https://travis-ci.org/ViViDboarder/docker-duplicity-cron)
+[![Build Status](https://travis-ci.org/ViViDboarder/docker-restic-cron.svg?branch=master)](https://travis-ci.org/ViViDboarder/docker-restic-cron)

 ## Instructions
 Mount any directories you'd like to back up as a volume and run
@ -8,43 +8,25 @@ Mount any directories you'd like to back up as a volume and run
 ## Env Variables
 | Variable | Default | Description |
 | -------- | ------- | ----------- |
-|AWS_ACCESS_KEY_ID| |Required for writing to S3|
-|AWS_DEFAULT_REGION| |Required for writing to S3|
-|AWS_SECRET_ACCESS_KEY| |Required for writing to S3|
-|BACKUP_DEST|file:///backups|Destination to store backups (See [duplicity documenation](http://duplicity.nongnu.org/duplicity.1.html#sect7))|
-|BACKUP_NAME|backup|What the name for the backup should be. If using a single store for multiple backups, make sure this is unique|
-|CLEANUP_COMMAND| |An optional duplicity command to execute after backups to clean older ones out (eg. "remove-all-but-n-full 2")|
+|AWS_ACCESS_KEY_ID| |Required for writing to S3 or Minio|
+|AWS_SECRET_ACCESS_KEY| |Required for writing to S3 or Minio|
+|B2_ACCOUNT_ID| |Required for writing to B2|
+|B2_ACCOUNT_KEY| |Required for writing to B2|
+|BACKUP_DEST|/backups|Destination to store backups (See [restic documenation](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html))|
+|CLEANUP_COMMAND| |Optional restic arguments for `forget` to execute after backups to clean older ones out (eg. "--prune --keep-last 2"). See [forget](https://restic.readthedocs.io/en/latest/060_forget.html)|
 |CRON_SCHEDULE| |If you want to periodic incremental backups on a schedule, provide it here. By default we just backup once and exit|
-|FLOCK_WAIT|60|Seconds to wait for a lock before skipping a backup|
-|FTP_PASSWORD| |Used to provide passwords for some backends. May not work without an attached TTY|
 |FULL_CRON_SCHEDULE| |If you want to periodic full backups on a schedule, provide it here. This requires an incremental cron schedule too|
-|GPG_KEY_ID| |The ID of the key you wish to use. See [Encryption](#encryption) section below|
-|OPT_ARGUMENTS| |Any additional arguments to provide to the duplicity backup command|
-|PASSPHRASE|Correct.Horse.Battery.Staple|Passphrase to use for GPG|
-|PATH_TO_BACKUP|/data|The path to the directory you wish to backup. If you want to backup multiple, see the [tip below](#backing-up-more-than-one-source-directory)|
+|OPT_ARGUMENTS| |Any additional arguments to provide to the restic command|
+|RESTIC_PASSWORD| |Passphrase to use for encryption|
+|PATH_TO_BACKUP|/data|The path to the directory you wish to backup|
 |RESTORE_ON_EMPTY_START| |Set this to "true" and if the `$PATH_TO_BACKUP` is empty, it will restore the latest backup. This can be used for auto recovery from lost data|
 |SKIP_ON_START| |Skips backup on start if set to "true"|
 |VERIFY_CRON_SCHEDULE| |If you want to verify your backups on a schedule, provide it here|

-## Encryption
-By default Duplicity will use a symettric encryption using just your passphrase. If you wish to use a GPG key, you can add a ro mount to your `~/.gnupg` directory and then provide the `GPG_KEY_ID` as an environment variable. The key will be used to sign and encrypt your files before sending to the backup destination.
-
-Need to generate a key? Install `gnupg` and run `gnupg --gen-key`
-
 ## Tips

-### Missing dependencies?
-Please file a ticket! Duplicity supports a ton of backends and I haven't had a chance to validate that all dependencies are present in the image. If something is missing, let me know and I'll add it
-
-### Getting complains about no terminal for askpass?
-Instead of using `FTP_PASSWORD`, add the password to the endpoint url
-
-### Backing up more than one source directory
-Duplicity only accepts one target, however you can refine that selection with `--exclude` and `--include` arguments. The below example shows how this can be used to select multiple backup sources
-```
-OPT_ARGUMENTS="--include /home --include /etc --exclude '**'"
-PATH_TO_BACKUP="/"
-```
+## Hostnames
+Hostname is used for identifying what you are backing up. You may want to specify this on your container.

 ### Backing up from another container
 Mount all volumes from your existing container with `--volumes-from` and then back up by providing the paths to those volumes. If there are more than one volumes, you'll want to use the above tip for mulitple backup sources
--- a/backup.sh
+++ b/backup.sh
@ -1,25 +1,15 @@
 #! /bin/bash
 set -e

-(
-    if ! flock -x -w $FLOCK_WAIT 200 ; then
-        echo 'ERROR: Could not obtain lock. Exiting.'
-        exit 1
-    fi
+restic \
+    -r $BACKUP_DEST \
+    $OPT_ARGUMENTS \
+    backup \
+    $PATH_TO_BACKUP

-    duplicity \
-        $1 \
-        --asynchronous-upload \
-        --log-file /root/duplicity.log \
-        --name $BACKUP_NAME \
-        $OPT_ARGUMENTS \
-        $PATH_TO_BACKUP \
-        $BACKUP_DEST
-
-    if [ -n "$CLEANUP_COMMAND" ]; then
-        duplicity $CLEANUP_COMMAND \
-            --log-file /root/duplicity.log \
-            --name $BACKUP_NAME \
-            $BACKUP_DEST
-    fi
-) 200>/var/lock/duplicity/.duplicity.lock
+if [ -n "$CLEANUP_COMMAND" ]; then
+    restic \
+        -r $BACKUP_DEST \
+        forget \
+        $CLEANUP_COMMAND
+fi
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@ -1,9 +1,9 @@
 version: '2'
 services:
-  duplicity:
+  restic:
    build:
      context: .
-      dockerfile: Dockerfile.ubuntu
+      dockerfile: Dockerfile
    volumes:
      - ./my-backups:/backups
      - ./my-data:/data:ro
--- a/restore.sh
+++ b/restore.sh
@ -1,18 +1,11 @@
 #! /bin/bash
 set -e

-(
-    if ! flock -x -w $FLOCK_WAIT 200 ; then
-        echo 'ERROR: Could not obtain lock. Exiting.'
-        exit 1
-    fi
+restore_snapshot=$1

-    duplicity restore \
-        --force \
-        --log-file /root/duplicity.log \
-        --name $BACKUP_NAME \
-        $OPT_ARGUMENTS \
-        $@ \
-        $BACKUP_DEST \
-        $PATH_TO_BACKUP
-) 200>/var/lock/duplicity/.duplicity.lock
+restic \
+    -r $BACKUP_DEST \
+    $OPT_ARGUMENTS \
+    restore \
+    $restore_snapshot \
+    -t /
--- a/start.sh
+++ b/start.sh
@ -11,14 +11,12 @@ if [ "$OPT_ARGUMENTS" == "" ]; then
    export OPT_ARGUMENTS="$@"
 fi

-# If key id is provied add arg
-if [ -e "$GPG_KEY_ID" ]; then
-    export OPT_ARGUMENTS="$OPT_ARGUMENTS --encrypt-sign-key=\"$GPG_KEY_ID\""
-fi
+# Init the repo
+restic -r $BACKUP_DEST snapshots || restic -r $BACKUP_DEST init

 # If set to restore on start, restore if the data volume is empty
 if [ "$RESTORE_ON_EMPTY_START" == "true" -a -z "$(ls -A $PATH_TO_BACKUP)" ]; then
-    /restore.sh
+    /restore.sh latest
 fi

 # Unless explicitly skipping, take a backup on startup
@ -39,11 +37,6 @@ if [ -n "$CRON_SCHEDULE" ]; then
    echo "$CRON_SCHEDULE source /env.sh && /backup.sh 2>> /cron.log" >> /crontab.conf
    echo "Backups scheduled as $CRON_SCHEDULE"

-    if [ -n "$FULL_CRON_SCHEDULE" ]; then
-        echo "$FULL_CRON_SCHEDULE source /env.sh && /backup.sh full 2>> /cron.log" >> /crontab.conf
-        echo "Full backup scheduled as $VERIFY_CRON_SCHEDULE"
-    fi
-
    if [ -n "$VERIFY_CRON_SCHEDULE" ]; then
        echo "$VERIFY_CRON_SCHEDULE source /env.sh && /verify.sh 2>> /cron.log" >> /crontab.conf
        echo "Verify scheduled as $VERIFY_CRON_SCHEDULE"
@ -52,9 +45,9 @@ if [ -n "$CRON_SCHEDULE" ]; then
    # Add to crontab
    crontab /crontab.conf

-    echo "Starting duplicity cron..."
+    echo "Starting restic cron..."
    cron

-    touch /cron.log /root/duplicity.log
-    tail -f /cron.log /root/duplicity.log
+    touch /cron.log
+    tail -f /cron.log
 fi
--- a/tests/docker-compose-test-s3.yml
+++ b/tests/docker-compose-test-s3.yml
@ -1,17 +1,17 @@
 version: '2'
 services:
-  duplicity:
+  restic:
    build:
      context: ..
-      dockerfile: Dockerfile.${DOCKER_BASE}
+      dockerfile: Dockerfile
    entrypoint: "bash"
-    command: ["/test.sh"]
+    command: "-c 'sleep 2 && /test.sh'"
    hostname: itest
    environment:
      IN_CONTAINER: 'true'
      SKIP_ON_START: 'true'
-      OPT_ARGUMENTS: '--s3-unencrypted-connection'
-      BACKUP_DEST: s3://minio:9000/duplicity
+      RESTIC_PASSWORD: Correct.Horse.Battery.Staple
+      BACKUP_DEST: s3:http://minio:9000/restic
      AWS_DEFAULT_REGION: us-east-1
      AWS_ACCESS_KEY_ID: SUPER_SECRET_ACCESS_KEY
      AWS_SECRET_ACCESS_KEY: SUPER_SECRET_SECRET_KEY
--- a/tests/test.sh
+++ b/tests/test.sh
@ -1,4 +1,5 @@
 #! /bin/bash
+set -e

 image=$1

@ -7,6 +8,7 @@ if [ "$IN_CONTAINER" != "true" ] ; then
    docker run --rm \
        -e IN_CONTAINER=true \
        -e SKIP_ON_START=true \
+        -e RESTIC_PASSWORD="Correct.Horse.Battery.Staple" \
        -v "$(pwd)/test.sh:/test.sh" \
        $image \
        bash -c "/test.sh"
@ -20,6 +22,9 @@ else
    echo "Create test data..."
    mkdir -p /data && echo Test > /data/test.txt

+    echo "Fake a start and init repo"
+    CRON_SCHEDULE="" /start.sh
+
    echo "Making backup..."
    /backup.sh

@ -33,7 +38,10 @@ else
    test -f /data/test.txt && exit 1 || echo "Gone"

    echo "Restore backup..."
-    /restore.sh
+    /restore.sh latest
+
+    echo "Verify restore..."
+    cat /data/test.txt

    echo "Verify backup..."
    /verify.sh
@ -48,11 +56,9 @@ else
    RESTORE_ON_EMPTY_START=true /start.sh

    echo "Verify restore happened..."
-    test -f /data/test.txt
+    cat /data/test.txt

    echo "Verify restore with incorrect passphrase fails..."
-    export PASSPHRASE=Incorrect.Mule.Solar.Paperclip
-
    echo "Fail to restore backup..."
-    /restore.sh && exit 1 || echo "OK"
+    RESTIC_PASSWORD=Incorrect.Mule.Solar.Paperclip /restore.sh latest && exit 1 || echo "OK"
 fi
--- a/verify.sh
+++ b/verify.sh
@ -1,10 +1,7 @@
 #! /bin/bash
 set -e

-duplicity verify \
-    --compare-data \
-    --log-file /root/duplicity.log \
-    --name $BACKUP_NAME \
+restic \
+    -r $BACKUP_DEST \
    $OPT_ARGUMENTS \
-    $BACKUP_DEST \
-    $PATH_TO_BACKUP
+    check