Let's get restic in here

Switch to restic
This commit is contained in:
ViViDboarder 2018-07-24 08:38:22 -07:00
parent 947dc939f2
commit 7e57f2ed0b
12 changed files with 90 additions and 204 deletions

27
Dockerfile Normal file
View File

@ -0,0 +1,27 @@
FROM ubuntu:artful
MAINTAINER ViViDboarder <vividboarder@gmail.com>
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
cron \
restic \
&& apt-get clean \
&& rm -rf /var/apt/lists/*
VOLUME /root/.cache/restic
VOLUME /backups
ENV BACKUP_DEST="/backups"
ENV BACKUP_NAME="backup"
ENV PATH_TO_BACKUP="/data"
# Cron schedules
ENV CRON_SCHEDULE=""
ENV VERIFY_CRON_SCHEDULE=""
ADD backup.sh /
ADD restore.sh /
ADD start.sh /
ADD verify.sh /
CMD [ "/start.sh" ]

View File

@ -1,51 +0,0 @@
FROM resin/rpi-raspbian:jessie
MAINTAINER ViViDboarder <vividboarder@gmail.com>
RUN [ "cross-build-start" ]
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
cron \
duplicity \
lftp \
ncftp \
openssh-client \
python-cloudfiles \
python-gdata \
python-oauthlib \
python-paramiko \
python-pexpect \
python-pip \
python-setuptools \
python-urllib3 \
rsync \
tahoe-lafs \
&& pip install -U boto b2 \
&& apt-get remove -y python-pip \
&& apt-get install -y --no-install-recommends \
python-swiftclient \
&& rm -rf /var/apt/lists/*
RUN [ "cross-build-end" ]
VOLUME /root/.cache/duplicity
VOLUME /backups
VOLUME /var/lock/duplicity
ENV BACKUP_DEST="file:///backups"
ENV BACKUP_NAME="backup"
ENV PATH_TO_BACKUP="/data"
ENV PASSPHRASE="Correct.Horse.Battery.Staple"
ENV FLOCK_WAIT=60
# Cron schedules
ENV CRON_SCHEDULE=""
ENV FULL_CRON_SCHEDULE=""
ENV VERIFY_CRON_SCHEDULE=""
ADD backup.sh /
ADD restore.sh /
ADD start.sh /
ADD verify.sh /
CMD [ "/start.sh" ]

View File

@ -1,51 +0,0 @@
FROM ubuntu:xenial
MAINTAINER ViViDboarder <vividboarder@gmail.com>
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
software-properties-common python-software-properties \
&& add-apt-repository ppa:duplicity-team/ppa \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
cron \
duplicity \
lftp \
ncftp \
openssh-client \
python-cloudfiles \
python-gdata \
python-oauthlib \
python-paramiko \
python-pexpect \
python-pip \
python-setuptools \
python-swiftclient \
python-urllib3 \
rsync \
tahoe-lafs \
&& pip install -U boto b2 \
&& apt-get autoremove -y python-pip \
&& apt-get clean \
&& rm -rf /var/apt/lists/*
VOLUME /root/.cache/duplicity
VOLUME /backups
VOLUME /var/lock/duplicity
ENV BACKUP_DEST="file:///backups"
ENV BACKUP_NAME="backup"
ENV PATH_TO_BACKUP="/data"
ENV PASSPHRASE="Correct.Horse.Battery.Staple"
ENV FLOCK_WAIT=60
# Cron schedules
ENV CRON_SCHEDULE=""
ENV FULL_CRON_SCHEDULE=""
ENV VERIFY_CRON_SCHEDULE=""
ADD backup.sh /
ADD restore.sh /
ADD start.sh /
ADD verify.sh /
CMD [ "/start.sh" ]

View File

@ -1,4 +1,4 @@
DOCKER_TAG ?= docker-duplicity-cron
DOCKER_TAG ?= docker-restic-cron
.PHONY: default
default: build-x86
@ -8,18 +8,18 @@ test: test-x86
.PHONY: build-x86
build-x86:
docker build -f ./Dockerfile.ubuntu -t $(DOCKER_TAG):ubuntu .
docker build -f ./Dockerfile -t $(DOCKER_TAG) --platform linux .
.PHONY: build-arm
build-arm:
docker build -f ./Dockerfile.raspbian -t $(DOCKER_TAG):raspbian .
docker build -f ./Dockerfile -t $(DOCKER_TAG) --platform arm .
.PHONY: build-all
build-all: build-x86 build-arm
.PHONY: test-x86
test-x86: build-x86
cd tests && ./test.sh $(DOCKER_TAG):ubuntu
cd tests && ./test.sh $(DOCKER_TAG)
.PHONY: test-arm
test-arm: build-arm
@ -41,7 +41,7 @@ test-s3-all: test-s3-x86 test-s3-arm
.PHONY: shell-x86
shell-x86: build-x86
docker run --rm -it $(DOCKER_TAG):ubuntu bash
docker run --rm -it $(DOCKER_TAG) bash
.PHONY: shell-arm
shell-arm: build-arm

View File

@ -1,6 +1,6 @@
# Duplicity Backup
# Restic Backup
[![Build Status](https://travis-ci.org/ViViDboarder/docker-duplicity-cron.svg?branch=master)](https://travis-ci.org/ViViDboarder/docker-duplicity-cron)
[![Build Status](https://travis-ci.org/ViViDboarder/docker-restic-cron.svg?branch=master)](https://travis-ci.org/ViViDboarder/docker-restic-cron)
## Instructions
Mount any directories you'd like to back up as a volume and run
@ -8,43 +8,25 @@ Mount any directories you'd like to back up as a volume and run
## Env Variables
| Variable | Default | Description |
| -------- | ------- | ----------- |
|AWS_ACCESS_KEY_ID| |Required for writing to S3|
|AWS_DEFAULT_REGION| |Required for writing to S3|
|AWS_SECRET_ACCESS_KEY| |Required for writing to S3|
|BACKUP_DEST|file:///backups|Destination to store backups (See [duplicity documenation](http://duplicity.nongnu.org/duplicity.1.html#sect7))|
|BACKUP_NAME|backup|What the name for the backup should be. If using a single store for multiple backups, make sure this is unique|
|CLEANUP_COMMAND| |An optional duplicity command to execute after backups to clean older ones out (eg. "remove-all-but-n-full 2")|
|AWS_ACCESS_KEY_ID| |Required for writing to S3 or Minio|
|AWS_SECRET_ACCESS_KEY| |Required for writing to S3 or Minio|
|B2_ACCOUNT_ID| |Required for writing to B2|
|B2_ACCOUNT_KEY| |Required for writing to B2|
|BACKUP_DEST|/backups|Destination to store backups (See [restic documenation](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html))|
|CLEANUP_COMMAND| |Optional restic arguments for `forget` to execute after backups to clean older ones out (eg. "--prune --keep-last 2"). See [forget](https://restic.readthedocs.io/en/latest/060_forget.html)|
|CRON_SCHEDULE| |If you want to periodic incremental backups on a schedule, provide it here. By default we just backup once and exit|
|FLOCK_WAIT|60|Seconds to wait for a lock before skipping a backup|
|FTP_PASSWORD| |Used to provide passwords for some backends. May not work without an attached TTY|
|FULL_CRON_SCHEDULE| |If you want to periodic full backups on a schedule, provide it here. This requires an incremental cron schedule too|
|GPG_KEY_ID| |The ID of the key you wish to use. See [Encryption](#encryption) section below|
|OPT_ARGUMENTS| |Any additional arguments to provide to the duplicity backup command|
|PASSPHRASE|Correct.Horse.Battery.Staple|Passphrase to use for GPG|
|PATH_TO_BACKUP|/data|The path to the directory you wish to backup. If you want to backup multiple, see the [tip below](#backing-up-more-than-one-source-directory)|
|OPT_ARGUMENTS| |Any additional arguments to provide to the restic command|
|RESTIC_PASSWORD| |Passphrase to use for encryption|
|PATH_TO_BACKUP|/data|The path to the directory you wish to backup|
|RESTORE_ON_EMPTY_START| |Set this to "true" and if the `$PATH_TO_BACKUP` is empty, it will restore the latest backup. This can be used for auto recovery from lost data|
|SKIP_ON_START| |Skips backup on start if set to "true"|
|VERIFY_CRON_SCHEDULE| |If you want to verify your backups on a schedule, provide it here|
## Encryption
By default Duplicity will use a symettric encryption using just your passphrase. If you wish to use a GPG key, you can add a ro mount to your `~/.gnupg` directory and then provide the `GPG_KEY_ID` as an environment variable. The key will be used to sign and encrypt your files before sending to the backup destination.
Need to generate a key? Install `gnupg` and run `gnupg --gen-key`
## Tips
### Missing dependencies?
Please file a ticket! Duplicity supports a ton of backends and I haven't had a chance to validate that all dependencies are present in the image. If something is missing, let me know and I'll add it
### Getting complains about no terminal for askpass?
Instead of using `FTP_PASSWORD`, add the password to the endpoint url
### Backing up more than one source directory
Duplicity only accepts one target, however you can refine that selection with `--exclude` and `--include` arguments. The below example shows how this can be used to select multiple backup sources
```
OPT_ARGUMENTS="--include /home --include /etc --exclude '**'"
PATH_TO_BACKUP="/"
```
## Hostnames
Hostname is used for identifying what you are backing up. You may want to specify this on your container.
### Backing up from another container
Mount all volumes from your existing container with `--volumes-from` and then back up by providing the paths to those volumes. If there are more than one volumes, you'll want to use the above tip for mulitple backup sources

View File

@ -1,25 +1,15 @@
#! /bin/bash
set -e
(
if ! flock -x -w $FLOCK_WAIT 200 ; then
echo 'ERROR: Could not obtain lock. Exiting.'
exit 1
fi
duplicity \
$1 \
--asynchronous-upload \
--log-file /root/duplicity.log \
--name $BACKUP_NAME \
restic \
-r $BACKUP_DEST \
$OPT_ARGUMENTS \
$PATH_TO_BACKUP \
$BACKUP_DEST
backup \
$PATH_TO_BACKUP
if [ -n "$CLEANUP_COMMAND" ]; then
duplicity $CLEANUP_COMMAND \
--log-file /root/duplicity.log \
--name $BACKUP_NAME \
$BACKUP_DEST
restic \
-r $BACKUP_DEST \
forget \
$CLEANUP_COMMAND
fi
) 200>/var/lock/duplicity/.duplicity.lock

View File

@ -1,9 +1,9 @@
version: '2'
services:
duplicity:
restic:
build:
context: .
dockerfile: Dockerfile.ubuntu
dockerfile: Dockerfile
volumes:
- ./my-backups:/backups
- ./my-data:/data:ro

View File

@ -1,18 +1,11 @@
#! /bin/bash
set -e
(
if ! flock -x -w $FLOCK_WAIT 200 ; then
echo 'ERROR: Could not obtain lock. Exiting.'
exit 1
fi
restore_snapshot=$1
duplicity restore \
--force \
--log-file /root/duplicity.log \
--name $BACKUP_NAME \
restic \
-r $BACKUP_DEST \
$OPT_ARGUMENTS \
$@ \
$BACKUP_DEST \
$PATH_TO_BACKUP
) 200>/var/lock/duplicity/.duplicity.lock
restore \
$restore_snapshot \
-t /

View File

@ -11,14 +11,12 @@ if [ "$OPT_ARGUMENTS" == "" ]; then
export OPT_ARGUMENTS="$@"
fi
# If key id is provied add arg
if [ -e "$GPG_KEY_ID" ]; then
export OPT_ARGUMENTS="$OPT_ARGUMENTS --encrypt-sign-key=\"$GPG_KEY_ID\""
fi
# Init the repo
restic -r $BACKUP_DEST snapshots || restic -r $BACKUP_DEST init
# If set to restore on start, restore if the data volume is empty
if [ "$RESTORE_ON_EMPTY_START" == "true" -a -z "$(ls -A $PATH_TO_BACKUP)" ]; then
/restore.sh
/restore.sh latest
fi
# Unless explicitly skipping, take a backup on startup
@ -39,11 +37,6 @@ if [ -n "$CRON_SCHEDULE" ]; then
echo "$CRON_SCHEDULE source /env.sh && /backup.sh 2>> /cron.log" >> /crontab.conf
echo "Backups scheduled as $CRON_SCHEDULE"
if [ -n "$FULL_CRON_SCHEDULE" ]; then
echo "$FULL_CRON_SCHEDULE source /env.sh && /backup.sh full 2>> /cron.log" >> /crontab.conf
echo "Full backup scheduled as $VERIFY_CRON_SCHEDULE"
fi
if [ -n "$VERIFY_CRON_SCHEDULE" ]; then
echo "$VERIFY_CRON_SCHEDULE source /env.sh && /verify.sh 2>> /cron.log" >> /crontab.conf
echo "Verify scheduled as $VERIFY_CRON_SCHEDULE"
@ -52,9 +45,9 @@ if [ -n "$CRON_SCHEDULE" ]; then
# Add to crontab
crontab /crontab.conf
echo "Starting duplicity cron..."
echo "Starting restic cron..."
cron
touch /cron.log /root/duplicity.log
tail -f /cron.log /root/duplicity.log
touch /cron.log
tail -f /cron.log
fi

View File

@ -1,17 +1,17 @@
version: '2'
services:
duplicity:
restic:
build:
context: ..
dockerfile: Dockerfile.${DOCKER_BASE}
dockerfile: Dockerfile
entrypoint: "bash"
command: ["/test.sh"]
command: "-c 'sleep 2 && /test.sh'"
hostname: itest
environment:
IN_CONTAINER: 'true'
SKIP_ON_START: 'true'
OPT_ARGUMENTS: '--s3-unencrypted-connection'
BACKUP_DEST: s3://minio:9000/duplicity
RESTIC_PASSWORD: Correct.Horse.Battery.Staple
BACKUP_DEST: s3:http://minio:9000/restic
AWS_DEFAULT_REGION: us-east-1
AWS_ACCESS_KEY_ID: SUPER_SECRET_ACCESS_KEY
AWS_SECRET_ACCESS_KEY: SUPER_SECRET_SECRET_KEY

View File

@ -1,4 +1,5 @@
#! /bin/bash
set -e
image=$1
@ -7,6 +8,7 @@ if [ "$IN_CONTAINER" != "true" ] ; then
docker run --rm \
-e IN_CONTAINER=true \
-e SKIP_ON_START=true \
-e RESTIC_PASSWORD="Correct.Horse.Battery.Staple" \
-v "$(pwd)/test.sh:/test.sh" \
$image \
bash -c "/test.sh"
@ -20,6 +22,9 @@ else
echo "Create test data..."
mkdir -p /data && echo Test > /data/test.txt
echo "Fake a start and init repo"
CRON_SCHEDULE="" /start.sh
echo "Making backup..."
/backup.sh
@ -33,7 +38,10 @@ else
test -f /data/test.txt && exit 1 || echo "Gone"
echo "Restore backup..."
/restore.sh
/restore.sh latest
echo "Verify restore..."
cat /data/test.txt
echo "Verify backup..."
/verify.sh
@ -48,11 +56,9 @@ else
RESTORE_ON_EMPTY_START=true /start.sh
echo "Verify restore happened..."
test -f /data/test.txt
cat /data/test.txt
echo "Verify restore with incorrect passphrase fails..."
export PASSPHRASE=Incorrect.Mule.Solar.Paperclip
echo "Fail to restore backup..."
/restore.sh && exit 1 || echo "OK"
RESTIC_PASSWORD=Incorrect.Mule.Solar.Paperclip /restore.sh latest && exit 1 || echo "OK"
fi

View File

@ -1,10 +1,7 @@
#! /bin/bash
set -e
duplicity verify \
--compare-data \
--log-file /root/duplicity.log \
--name $BACKUP_NAME \
restic \
-r $BACKUP_DEST \
$OPT_ARGUMENTS \
$BACKUP_DEST \
$PATH_TO_BACKUP
check