Let's get restic in here

Switch to restic
This commit is contained in:
ViViDboarder 2018-07-24 08:38:22 -07:00
parent 947dc939f2
commit 7e57f2ed0b
12 changed files with 90 additions and 204 deletions

27
Dockerfile Normal file
View File

@ -0,0 +1,27 @@
FROM ubuntu:artful
MAINTAINER ViViDboarder <vividboarder@gmail.com>
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
cron \
restic \
&& apt-get clean \
&& rm -rf /var/apt/lists/*
VOLUME /root/.cache/restic
VOLUME /backups
ENV BACKUP_DEST="/backups"
ENV BACKUP_NAME="backup"
ENV PATH_TO_BACKUP="/data"
# Cron schedules
ENV CRON_SCHEDULE=""
ENV VERIFY_CRON_SCHEDULE=""
ADD backup.sh /
ADD restore.sh /
ADD start.sh /
ADD verify.sh /
CMD [ "/start.sh" ]

View File

@ -1,51 +0,0 @@
FROM resin/rpi-raspbian:jessie
MAINTAINER ViViDboarder <vividboarder@gmail.com>
RUN [ "cross-build-start" ]
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
cron \
duplicity \
lftp \
ncftp \
openssh-client \
python-cloudfiles \
python-gdata \
python-oauthlib \
python-paramiko \
python-pexpect \
python-pip \
python-setuptools \
python-urllib3 \
rsync \
tahoe-lafs \
&& pip install -U boto b2 \
&& apt-get remove -y python-pip \
&& apt-get install -y --no-install-recommends \
python-swiftclient \
&& rm -rf /var/apt/lists/*
RUN [ "cross-build-end" ]
VOLUME /root/.cache/duplicity
VOLUME /backups
VOLUME /var/lock/duplicity
ENV BACKUP_DEST="file:///backups"
ENV BACKUP_NAME="backup"
ENV PATH_TO_BACKUP="/data"
ENV PASSPHRASE="Correct.Horse.Battery.Staple"
ENV FLOCK_WAIT=60
# Cron schedules
ENV CRON_SCHEDULE=""
ENV FULL_CRON_SCHEDULE=""
ENV VERIFY_CRON_SCHEDULE=""
ADD backup.sh /
ADD restore.sh /
ADD start.sh /
ADD verify.sh /
CMD [ "/start.sh" ]

View File

@ -1,51 +0,0 @@
FROM ubuntu:xenial
MAINTAINER ViViDboarder <vividboarder@gmail.com>
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
software-properties-common python-software-properties \
&& add-apt-repository ppa:duplicity-team/ppa \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
cron \
duplicity \
lftp \
ncftp \
openssh-client \
python-cloudfiles \
python-gdata \
python-oauthlib \
python-paramiko \
python-pexpect \
python-pip \
python-setuptools \
python-swiftclient \
python-urllib3 \
rsync \
tahoe-lafs \
&& pip install -U boto b2 \
&& apt-get autoremove -y python-pip \
&& apt-get clean \
&& rm -rf /var/apt/lists/*
VOLUME /root/.cache/duplicity
VOLUME /backups
VOLUME /var/lock/duplicity
ENV BACKUP_DEST="file:///backups"
ENV BACKUP_NAME="backup"
ENV PATH_TO_BACKUP="/data"
ENV PASSPHRASE="Correct.Horse.Battery.Staple"
ENV FLOCK_WAIT=60
# Cron schedules
ENV CRON_SCHEDULE=""
ENV FULL_CRON_SCHEDULE=""
ENV VERIFY_CRON_SCHEDULE=""
ADD backup.sh /
ADD restore.sh /
ADD start.sh /
ADD verify.sh /
CMD [ "/start.sh" ]

View File

@ -1,4 +1,4 @@
DOCKER_TAG ?= docker-duplicity-cron DOCKER_TAG ?= docker-restic-cron
.PHONY: default .PHONY: default
default: build-x86 default: build-x86
@ -8,18 +8,18 @@ test: test-x86
.PHONY: build-x86 .PHONY: build-x86
build-x86: build-x86:
docker build -f ./Dockerfile.ubuntu -t $(DOCKER_TAG):ubuntu . docker build -f ./Dockerfile -t $(DOCKER_TAG) --platform linux .
.PHONY: build-arm .PHONY: build-arm
build-arm: build-arm:
docker build -f ./Dockerfile.raspbian -t $(DOCKER_TAG):raspbian . docker build -f ./Dockerfile -t $(DOCKER_TAG) --platform arm .
.PHONY: build-all .PHONY: build-all
build-all: build-x86 build-arm build-all: build-x86 build-arm
.PHONY: test-x86 .PHONY: test-x86
test-x86: build-x86 test-x86: build-x86
cd tests && ./test.sh $(DOCKER_TAG):ubuntu cd tests && ./test.sh $(DOCKER_TAG)
.PHONY: test-arm .PHONY: test-arm
test-arm: build-arm test-arm: build-arm
@ -41,7 +41,7 @@ test-s3-all: test-s3-x86 test-s3-arm
.PHONY: shell-x86 .PHONY: shell-x86
shell-x86: build-x86 shell-x86: build-x86
docker run --rm -it $(DOCKER_TAG):ubuntu bash docker run --rm -it $(DOCKER_TAG) bash
.PHONY: shell-arm .PHONY: shell-arm
shell-arm: build-arm shell-arm: build-arm

View File

@ -1,6 +1,6 @@
# Duplicity Backup # Restic Backup
[![Build Status](https://travis-ci.org/ViViDboarder/docker-duplicity-cron.svg?branch=master)](https://travis-ci.org/ViViDboarder/docker-duplicity-cron) [![Build Status](https://travis-ci.org/ViViDboarder/docker-restic-cron.svg?branch=master)](https://travis-ci.org/ViViDboarder/docker-restic-cron)
## Instructions ## Instructions
Mount any directories you'd like to back up as a volume and run Mount any directories you'd like to back up as a volume and run
@ -8,43 +8,25 @@ Mount any directories you'd like to back up as a volume and run
## Env Variables ## Env Variables
| Variable | Default | Description | | Variable | Default | Description |
| -------- | ------- | ----------- | | -------- | ------- | ----------- |
|AWS_ACCESS_KEY_ID| |Required for writing to S3| |AWS_ACCESS_KEY_ID| |Required for writing to S3 or Minio|
|AWS_DEFAULT_REGION| |Required for writing to S3| |AWS_SECRET_ACCESS_KEY| |Required for writing to S3 or Minio|
|AWS_SECRET_ACCESS_KEY| |Required for writing to S3| |B2_ACCOUNT_ID| |Required for writing to B2|
|BACKUP_DEST|file:///backups|Destination to store backups (See [duplicity documenation](http://duplicity.nongnu.org/duplicity.1.html#sect7))| |B2_ACCOUNT_KEY| |Required for writing to B2|
|BACKUP_NAME|backup|What the name for the backup should be. If using a single store for multiple backups, make sure this is unique| |BACKUP_DEST|/backups|Destination to store backups (See [restic documenation](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html))|
|CLEANUP_COMMAND| |An optional duplicity command to execute after backups to clean older ones out (eg. "remove-all-but-n-full 2")| |CLEANUP_COMMAND| |Optional restic arguments for `forget` to execute after backups to clean older ones out (eg. "--prune --keep-last 2"). See [forget](https://restic.readthedocs.io/en/latest/060_forget.html)|
|CRON_SCHEDULE| |If you want to periodic incremental backups on a schedule, provide it here. By default we just backup once and exit| |CRON_SCHEDULE| |If you want to periodic incremental backups on a schedule, provide it here. By default we just backup once and exit|
|FLOCK_WAIT|60|Seconds to wait for a lock before skipping a backup|
|FTP_PASSWORD| |Used to provide passwords for some backends. May not work without an attached TTY|
|FULL_CRON_SCHEDULE| |If you want to periodic full backups on a schedule, provide it here. This requires an incremental cron schedule too| |FULL_CRON_SCHEDULE| |If you want to periodic full backups on a schedule, provide it here. This requires an incremental cron schedule too|
|GPG_KEY_ID| |The ID of the key you wish to use. See [Encryption](#encryption) section below| |OPT_ARGUMENTS| |Any additional arguments to provide to the restic command|
|OPT_ARGUMENTS| |Any additional arguments to provide to the duplicity backup command| |RESTIC_PASSWORD| |Passphrase to use for encryption|
|PASSPHRASE|Correct.Horse.Battery.Staple|Passphrase to use for GPG| |PATH_TO_BACKUP|/data|The path to the directory you wish to backup|
|PATH_TO_BACKUP|/data|The path to the directory you wish to backup. If you want to backup multiple, see the [tip below](#backing-up-more-than-one-source-directory)|
|RESTORE_ON_EMPTY_START| |Set this to "true" and if the `$PATH_TO_BACKUP` is empty, it will restore the latest backup. This can be used for auto recovery from lost data| |RESTORE_ON_EMPTY_START| |Set this to "true" and if the `$PATH_TO_BACKUP` is empty, it will restore the latest backup. This can be used for auto recovery from lost data|
|SKIP_ON_START| |Skips backup on start if set to "true"| |SKIP_ON_START| |Skips backup on start if set to "true"|
|VERIFY_CRON_SCHEDULE| |If you want to verify your backups on a schedule, provide it here| |VERIFY_CRON_SCHEDULE| |If you want to verify your backups on a schedule, provide it here|
## Encryption
By default Duplicity will use a symettric encryption using just your passphrase. If you wish to use a GPG key, you can add a ro mount to your `~/.gnupg` directory and then provide the `GPG_KEY_ID` as an environment variable. The key will be used to sign and encrypt your files before sending to the backup destination.
Need to generate a key? Install `gnupg` and run `gnupg --gen-key`
## Tips ## Tips
### Missing dependencies? ## Hostnames
Please file a ticket! Duplicity supports a ton of backends and I haven't had a chance to validate that all dependencies are present in the image. If something is missing, let me know and I'll add it Hostname is used for identifying what you are backing up. You may want to specify this on your container.
### Getting complains about no terminal for askpass?
Instead of using `FTP_PASSWORD`, add the password to the endpoint url
### Backing up more than one source directory
Duplicity only accepts one target, however you can refine that selection with `--exclude` and `--include` arguments. The below example shows how this can be used to select multiple backup sources
```
OPT_ARGUMENTS="--include /home --include /etc --exclude '**'"
PATH_TO_BACKUP="/"
```
### Backing up from another container ### Backing up from another container
Mount all volumes from your existing container with `--volumes-from` and then back up by providing the paths to those volumes. If there are more than one volumes, you'll want to use the above tip for mulitple backup sources Mount all volumes from your existing container with `--volumes-from` and then back up by providing the paths to those volumes. If there are more than one volumes, you'll want to use the above tip for mulitple backup sources

View File

@ -1,25 +1,15 @@
#! /bin/bash #! /bin/bash
set -e set -e
( restic \
if ! flock -x -w $FLOCK_WAIT 200 ; then -r $BACKUP_DEST \
echo 'ERROR: Could not obtain lock. Exiting.'
exit 1
fi
duplicity \
$1 \
--asynchronous-upload \
--log-file /root/duplicity.log \
--name $BACKUP_NAME \
$OPT_ARGUMENTS \ $OPT_ARGUMENTS \
$PATH_TO_BACKUP \ backup \
$BACKUP_DEST $PATH_TO_BACKUP
if [ -n "$CLEANUP_COMMAND" ]; then if [ -n "$CLEANUP_COMMAND" ]; then
duplicity $CLEANUP_COMMAND \ restic \
--log-file /root/duplicity.log \ -r $BACKUP_DEST \
--name $BACKUP_NAME \ forget \
$BACKUP_DEST $CLEANUP_COMMAND
fi fi
) 200>/var/lock/duplicity/.duplicity.lock

View File

@ -1,9 +1,9 @@
version: '2' version: '2'
services: services:
duplicity: restic:
build: build:
context: . context: .
dockerfile: Dockerfile.ubuntu dockerfile: Dockerfile
volumes: volumes:
- ./my-backups:/backups - ./my-backups:/backups
- ./my-data:/data:ro - ./my-data:/data:ro

View File

@ -1,18 +1,11 @@
#! /bin/bash #! /bin/bash
set -e set -e
( restore_snapshot=$1
if ! flock -x -w $FLOCK_WAIT 200 ; then
echo 'ERROR: Could not obtain lock. Exiting.'
exit 1
fi
duplicity restore \ restic \
--force \ -r $BACKUP_DEST \
--log-file /root/duplicity.log \
--name $BACKUP_NAME \
$OPT_ARGUMENTS \ $OPT_ARGUMENTS \
$@ \ restore \
$BACKUP_DEST \ $restore_snapshot \
$PATH_TO_BACKUP -t /
) 200>/var/lock/duplicity/.duplicity.lock

View File

@ -11,14 +11,12 @@ if [ "$OPT_ARGUMENTS" == "" ]; then
export OPT_ARGUMENTS="$@" export OPT_ARGUMENTS="$@"
fi fi
# If key id is provied add arg # Init the repo
if [ -e "$GPG_KEY_ID" ]; then restic -r $BACKUP_DEST snapshots || restic -r $BACKUP_DEST init
export OPT_ARGUMENTS="$OPT_ARGUMENTS --encrypt-sign-key=\"$GPG_KEY_ID\""
fi
# If set to restore on start, restore if the data volume is empty # If set to restore on start, restore if the data volume is empty
if [ "$RESTORE_ON_EMPTY_START" == "true" -a -z "$(ls -A $PATH_TO_BACKUP)" ]; then if [ "$RESTORE_ON_EMPTY_START" == "true" -a -z "$(ls -A $PATH_TO_BACKUP)" ]; then
/restore.sh /restore.sh latest
fi fi
# Unless explicitly skipping, take a backup on startup # Unless explicitly skipping, take a backup on startup
@ -39,11 +37,6 @@ if [ -n "$CRON_SCHEDULE" ]; then
echo "$CRON_SCHEDULE source /env.sh && /backup.sh 2>> /cron.log" >> /crontab.conf echo "$CRON_SCHEDULE source /env.sh && /backup.sh 2>> /cron.log" >> /crontab.conf
echo "Backups scheduled as $CRON_SCHEDULE" echo "Backups scheduled as $CRON_SCHEDULE"
if [ -n "$FULL_CRON_SCHEDULE" ]; then
echo "$FULL_CRON_SCHEDULE source /env.sh && /backup.sh full 2>> /cron.log" >> /crontab.conf
echo "Full backup scheduled as $VERIFY_CRON_SCHEDULE"
fi
if [ -n "$VERIFY_CRON_SCHEDULE" ]; then if [ -n "$VERIFY_CRON_SCHEDULE" ]; then
echo "$VERIFY_CRON_SCHEDULE source /env.sh && /verify.sh 2>> /cron.log" >> /crontab.conf echo "$VERIFY_CRON_SCHEDULE source /env.sh && /verify.sh 2>> /cron.log" >> /crontab.conf
echo "Verify scheduled as $VERIFY_CRON_SCHEDULE" echo "Verify scheduled as $VERIFY_CRON_SCHEDULE"
@ -52,9 +45,9 @@ if [ -n "$CRON_SCHEDULE" ]; then
# Add to crontab # Add to crontab
crontab /crontab.conf crontab /crontab.conf
echo "Starting duplicity cron..." echo "Starting restic cron..."
cron cron
touch /cron.log /root/duplicity.log touch /cron.log
tail -f /cron.log /root/duplicity.log tail -f /cron.log
fi fi

View File

@ -1,17 +1,17 @@
version: '2' version: '2'
services: services:
duplicity: restic:
build: build:
context: .. context: ..
dockerfile: Dockerfile.${DOCKER_BASE} dockerfile: Dockerfile
entrypoint: "bash" entrypoint: "bash"
command: ["/test.sh"] command: "-c 'sleep 2 && /test.sh'"
hostname: itest hostname: itest
environment: environment:
IN_CONTAINER: 'true' IN_CONTAINER: 'true'
SKIP_ON_START: 'true' SKIP_ON_START: 'true'
OPT_ARGUMENTS: '--s3-unencrypted-connection' RESTIC_PASSWORD: Correct.Horse.Battery.Staple
BACKUP_DEST: s3://minio:9000/duplicity BACKUP_DEST: s3:http://minio:9000/restic
AWS_DEFAULT_REGION: us-east-1 AWS_DEFAULT_REGION: us-east-1
AWS_ACCESS_KEY_ID: SUPER_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID: SUPER_SECRET_ACCESS_KEY
AWS_SECRET_ACCESS_KEY: SUPER_SECRET_SECRET_KEY AWS_SECRET_ACCESS_KEY: SUPER_SECRET_SECRET_KEY

View File

@ -1,4 +1,5 @@
#! /bin/bash #! /bin/bash
set -e
image=$1 image=$1
@ -7,6 +8,7 @@ if [ "$IN_CONTAINER" != "true" ] ; then
docker run --rm \ docker run --rm \
-e IN_CONTAINER=true \ -e IN_CONTAINER=true \
-e SKIP_ON_START=true \ -e SKIP_ON_START=true \
-e RESTIC_PASSWORD="Correct.Horse.Battery.Staple" \
-v "$(pwd)/test.sh:/test.sh" \ -v "$(pwd)/test.sh:/test.sh" \
$image \ $image \
bash -c "/test.sh" bash -c "/test.sh"
@ -20,6 +22,9 @@ else
echo "Create test data..." echo "Create test data..."
mkdir -p /data && echo Test > /data/test.txt mkdir -p /data && echo Test > /data/test.txt
echo "Fake a start and init repo"
CRON_SCHEDULE="" /start.sh
echo "Making backup..." echo "Making backup..."
/backup.sh /backup.sh
@ -33,7 +38,10 @@ else
test -f /data/test.txt && exit 1 || echo "Gone" test -f /data/test.txt && exit 1 || echo "Gone"
echo "Restore backup..." echo "Restore backup..."
/restore.sh /restore.sh latest
echo "Verify restore..."
cat /data/test.txt
echo "Verify backup..." echo "Verify backup..."
/verify.sh /verify.sh
@ -48,11 +56,9 @@ else
RESTORE_ON_EMPTY_START=true /start.sh RESTORE_ON_EMPTY_START=true /start.sh
echo "Verify restore happened..." echo "Verify restore happened..."
test -f /data/test.txt cat /data/test.txt
echo "Verify restore with incorrect passphrase fails..." echo "Verify restore with incorrect passphrase fails..."
export PASSPHRASE=Incorrect.Mule.Solar.Paperclip
echo "Fail to restore backup..." echo "Fail to restore backup..."
/restore.sh && exit 1 || echo "OK" RESTIC_PASSWORD=Incorrect.Mule.Solar.Paperclip /restore.sh latest && exit 1 || echo "OK"
fi fi

View File

@ -1,10 +1,7 @@
#! /bin/bash #! /bin/bash
set -e set -e
duplicity verify \ restic \
--compare-data \ -r $BACKUP_DEST \
--log-file /root/duplicity.log \
--name $BACKUP_NAME \
$OPT_ARGUMENTS \ $OPT_ARGUMENTS \
$BACKUP_DEST \ check
$PATH_TO_BACKUP