Merge pull request #1 from GoliathLabs/master

Added: Dependabot.yml

.github/dependabot.yml

@@ -0,0 +1,16 @@
+# Docs: <https://docs.github.com/en/free-pro-team@latest/github/administering-a-repository/customizing-dependency-updates>
+
+version: 2
+
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule: {interval: monthly}
+    reviewers: [ViViDboarder]
+    assignees: [ViViDboarder]
+
+  - package-ecosystem: docker
+    directory: /
+    schedule: {interval: monthly}
+    reviewers: [ViViDboarder]
+    assignees: [ViViDboarder]

.github/workflows/docker.yml

@@ -0,0 +1,58 @@
+---
+name: push-docker-images
+
+"on":
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: vividboarder/docker-restic-cron
+          flavor: |
+            latest=auto
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/tests.yml

@@ -0,0 +1,30 @@
+---
+name: tests
+
+"on":
+  push:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Run base tests
+        run: make test-x86
+
+      - name: Run s3 tests
+        run: make test-s3-x86
+
+      - name: Setup python
+        uses: actions/setup-python@v2.2.2
+
+      - name: Run pre-commit hooks
+        run: |
+          pip install pre-commit
+          make check

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: check-merge-conflict
+      - id: check-yaml
+      - id: end-of-file-fixer
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.7.2.1
+    hooks:
+      - id: shellcheck
+  - repo: https://github.com/IamTheFij/docker-pre-commit
+    rev: v2.0.0
+    hooks:
+      - id: docker-compose-check
+      - id: hadolint

Dockerfile

@@ -1,33 +1,31 @@
 ARG REPO=library
-FROM ${REPO}/ubuntu:focal
+FROM ${REPO}/alpine:3.12
-LABEL maintainer="ViViDboarder <vividboarder@gmail.com>"
 
-RUN apt-get update \
+ARG TARGETARCH
-        && apt-get install -y --no-install-recommends \
-            ca-certificates \
-            cron \
-            restic=0.9.6* \
-        && apt-get clean \
-        && rm -rf /var/apt/lists/*
 
-VOLUME /root/.cache/restic
+RUN apk add --no-cache curl=~7 bash=~5
-VOLUME /backups
 
+ARG RCLONE_VERSION=v1.55.1
+
+COPY ./scripts/install_rclone.sh /scripts/
+RUN /scripts/install_rclone.sh "$RCLONE_VERSION" "$TARGETARCH"
+
+ARG RESTIC_VERSION=0.12.0
+
+COPY ./scripts/install_restic.sh /scripts/
+RUN /scripts/install_restic.sh "$RESTIC_VERSION" "$TARGETARCH"
+
+# Set some default environment variables
 ENV BACKUP_DEST="/backups"
 ENV BACKUP_NAME="backup"
 ENV PATH_TO_BACKUP="/data"
-
-# Cron schedules
 ENV CRON_SCHEDULE=""
 ENV VERIFY_CRON_SCHEDULE=""
 
-COPY backup.sh /
+COPY ./scripts /scripts
-COPY restore.sh /
-COPY start.sh /
-COPY verify.sh /
-COPY healthcheck.sh /
-COPY cron-exec.sh /
 
-HEALTHCHECK CMD /healthcheck.sh
+HEALTHCHECK CMD /scripts/healthcheck.sh
 
-CMD [ "/start.sh" ]
+VOLUME /root/.config
+
+CMD [ "/scripts/start.sh" ]

Makefile

@@ -1,4 +1,7 @@
-DOCKER_TAG ?= docker-restic-cron
+DOCKER_TAG ?= docker-restic-cron-$(USER)
+
+.PHONY: all
+all: check test-all
 
 .PHONY: default
 default: build-x86
@@ -8,11 +11,11 @@ test: test-x86
 
 .PHONY: build-x86
 build-x86:
-	docker build -f ./Dockerfile -t $(DOCKER_TAG) .
+	docker build --build-arg TARGETARCH=amd64 -f ./Dockerfile -t $(DOCKER_TAG) .
 
 .PHONY: build-arm
 build-arm:
-	docker build --build-arg REPO=arm32v7 -f ./Dockerfile -t $(DOCKER_TAG)-arm32v7 .
+	docker build --build-arg REPO=arm32v7 --build-arg TARGETARCH=arm -f ./Dockerfile -t $(DOCKER_TAG)-arm .
 
 .PHONY: build-all
 build-all: build-x86 build-arm
@@ -22,15 +25,17 @@ test-x86: build-x86
 	cd tests && ./test.sh $(DOCKER_TAG)
 	cd tests && ./test-pre-scripts.sh $(DOCKER_TAG)
 
+.PHONY: test-arm
+test-arm: build-arm
+	cd tests && ./test.sh $(DOCKER_TAG)-arm
+	cd tests && ./test-pre-scripts.sh $(DOCKER_TAG)-arm
+
 .PHONY: test-all
 test-all: test-x86 test-arm
 
 .PHONY: test-s3-x86
 test-s3-x86:
-	cd tests && ./test-s3.sh ubuntu
+	cd tests && env TARGETARCH=amd64 ./test-s3.sh
-
-.PHONY: test-s3-all
-test-s3-all: test-s3-x86 test-s3-arm
 
 .PHONY: shell-x86
 shell-x86: build-x86
@@ -41,4 +46,14 @@ shell: shell-x86
 
 .PHONY: clean
 clean:
-	docker-compose -f docker-compose-test-s3.yml down -v
+	docker-compose -f ./tests/docker-compose-test-s3.yml down -v
+	rm -fr my-backups/
+	rm -fr my-data/
+
+.PHONY: install-hooks
+install-hooks:
+	pre-commit install
+
+.PHONY: check
+check:
+	pre-commit run --all-files

Readme.md

@@ -31,7 +31,12 @@ Hostname is used for identifying what you are backing up. You may want to specif
 Mount all volumes from your existing container with `--volumes-from` and then back up by providing the paths to those volumes. If there are more than one volumes, you'll want to use the above tip for mulitple backup sources
 
 ### Restoring a backup
-On your running container, execute `/restore.sh`. That should be that! Eg. `docker exec my_backup_container /restore.sh`
+On your running container, execute `/scripts/restore.sh`. That should be that! Eg. `docker exec my_backup_container /scripts/restore.sh`
 
-### To Do
+### Backup to any rclone destination
- - [ ] Automatic restoration if there is no source data
+This image also contains [rclone](https://rclone.org). This allows you to target any destination supported by rclone. Check out the [official documentation](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html#other-services-via-rclone) to see how to configure this.
+
+Rather than having to use an exec shell inside the container, I recommend configuring via the backup destination. For example: `rclone::ftp,env_auth:/test-restic` and then passing all config values via the environment.
+
+### Pre/post backup/restore scripts
+Before and after any backup or restore, scripts located in `/scripts/{backup,restore}/{before,after}/` will be executed. This can be used to do things like snapshotting a database before backing it up and restoring the contents.

backup.sh

@@ -1,29 +0,0 @@
-#! /bin/bash
-set -e
-
-# Run pre-backup scripts
-for f in /scripts/backup/before/*; do
-    if [ -f $f -a -x $f ]; then
-        bash $f
-    fi
-done
-
-restic \
-    -r $BACKUP_DEST \
-    $OPT_ARGUMENTS \
-    backup \
-    "$PATH_TO_BACKUP"
-
-if [ -n "$CLEANUP_COMMAND" ]; then
-    restic \
-        -r $BACKUP_DEST \
-        forget \
-        $CLEANUP_COMMAND
-fi
-
-# Run post-backup scripts
-for f in /scripts/backup/after/*; do
-    if [ -f $f -a -x $f ]; then
-        bash $f
-    fi
-done

cron-exec.sh

@@ -1,11 +0,0 @@
-#! /bin/bash
-
-ENV=/env.sh
-LOG=/cron.log
-HEALTH_FILE=/unhealthy
-
-touch $ENV
-source $ENV
-
-# Execute command and write output to log
-$@ 2>> $LOG && rm -f $HEALTH_FILE || { touch $HEALTH_FILE; exit 1; }

scripts/backup.sh

@@ -0,0 +1,35 @@
+#! /bin/bash
+set -e
+
+# Run pre-backup scripts
+for f in /scripts/backup/before/*; do
+    if [ -f "$f" ] && [ -x "$f" ]; then
+        bash "$f"
+    fi
+done
+
+# shellcheck disable=SC2086
+restic \
+    -r "$BACKUP_DEST" \
+    $OPT_ARGUMENTS \
+    backup \
+    "$PATH_TO_BACKUP"
+
+if [ -n "$CLEANUP_COMMAND" ]; then
+    # Clean up old snapshots via provided policy
+    # shellcheck disable=SC2086
+    restic \
+        -r "$BACKUP_DEST" \
+        forget \
+        $CLEANUP_COMMAND
+
+    # Verify that nothing is corrupted
+    restic check -r "$BACKUP_DEST"
+fi
+
+# Run post-backup scripts
+for f in /scripts/backup/after/*; do
+    if [ -f "$f" ] && [ -x "$f" ]; then
+        bash "$f"
+    fi
+done

scripts/cron-exec.sh

@@ -0,0 +1,18 @@
+#! /bin/bash
+
+ENV=/env.sh
+LOG=/cron.log
+HEALTH_FILE=/unhealthy
+
+if [ -f "$ENV" ]; then
+    # shellcheck disable=SC1090
+    source "$ENV"
+fi
+
+# Execute command and write output to log
+if eval "$@" 2>> "$LOG"; then
+    rm -f "$HEALTH_FILE"
+else
+    touch "$HEALTH_FILE"
+    exit 1;
+fi

scripts/install_rclone.sh

@@ -0,0 +1,18 @@
+#! /bin/bash
+set -ex
+
+VERSION="$1"
+ARCH="$2"
+
+RCLONE_NAME=rclone-${VERSION}-linux-${ARCH}
+
+# Download
+curl -o rclone.zip "https://downloads.rclone.org/${VERSION}/${RCLONE_NAME}.zip"
+
+# Install
+unzip rclone.zip
+mv "${RCLONE_NAME}/rclone" /usr/local/bin/
+
+# Clean up
+rm rclone.zip
+rm -fr "${RCLONE_NAME}"

scripts/install_restic.sh

@@ -0,0 +1,15 @@
+#! /bin/bash
+set -ex
+
+VERSION="$1"
+ARCH="$2"
+
+RESTIC_NAME=restic_${VERSION}_linux_${ARCH}
+
+# Download
+curl -L -o restic.bz2 "https://github.com/restic/restic/releases/download/v${VERSION}/${RESTIC_NAME}.bz2"
+
+# Install
+bunzip2 -v restic.bz2
+mv restic /usr/local/bin/
+chmod +x /usr/local/bin/restic

scripts/restore.sh

@@ -1,25 +1,26 @@
 #! /bin/bash
 set -e
 
-restore_snapshot=$1
+restore_snapshot="$1"
 
 # Run pre-restore scripts
 for f in /scripts/restore/before/*; do
-    if [ -f $f -a -x $f ]; then
+    if [ -f "$f" ] && [ -x "$f" ]; then
-        bash $f
+        bash "$f"
     fi
 done
 
+# shellcheck disable=SC2086
 restic \
-    -r $BACKUP_DEST \
+    -r "$BACKUP_DEST" \
     $OPT_ARGUMENTS \
     restore \
-    $restore_snapshot \
+    "$restore_snapshot" \
     -t /
 
 # Run post-restore scripts
 for f in /scripts/restore/after/*; do
-    if [ -f $f -a -x $f ]; then
+    if [ -f "$f" ] && [ -x "$f" ]; then
-        bash $f
+        bash "$f"
     fi
 done

scripts/start.sh

@@ -8,20 +8,20 @@ fi
 
 # If no env variable set, get from command line
 if [ "$OPT_ARGUMENTS" == "" ]; then
-    export OPT_ARGUMENTS="$@"
+    export OPT_ARGUMENTS="$*"
 fi
 
 # Init the repo
-restic -r $BACKUP_DEST snapshots || restic -r $BACKUP_DEST init
+restic -r "$BACKUP_DEST" snapshots || restic -r "$BACKUP_DEST" init
 
 # If set to restore on start, restore if the data volume is empty
-if [ "$RESTORE_ON_EMPTY_START" == "true" -a -z "$(ls -A $PATH_TO_BACKUP)" ]; then
+if [ "$RESTORE_ON_EMPTY_START" == "true" ] && [ -z "$(ls -A "$PATH_TO_BACKUP")" ]; then
-    /cron-exec.sh /restore.sh latest
+    /scripts/cron-exec.sh /scripts/restore.sh latest
 fi
 
 # Unless explicitly skipping, take a backup on startup
 if [ "$SKIP_ON_START" != "true" ]; then
-    /cron-exec.sh /backup.sh
+    /scripts/cron-exec.sh /scripts/backup.sh
 fi
 
 if [ -n "$CRON_SCHEDULE" ]; then
@@ -34,19 +34,22 @@ if [ -n "$CRON_SCHEDULE" ]; then
     echo "SHELL=/bin/bash" > /crontab.conf
 
     # Schedule the backups
-    echo "$CRON_SCHEDULE /cron-exec.sh /backup.sh" >> /crontab.conf
+    echo "$CRON_SCHEDULE /scripts/cron-exec.sh /scripts/backup.sh" >> /crontab.conf
     echo "Backups scheduled as $CRON_SCHEDULE"
 
     if [ -n "$VERIFY_CRON_SCHEDULE" ]; then
-        echo "$VERIFY_CRON_SCHEDULE /cron-exec.sh /verify.sh" >> /crontab.conf
+        echo "$VERIFY_CRON_SCHEDULE /scripts/cron-exec.sh /scripts/verify.sh" >> /crontab.conf
         echo "Verify scheduled as $VERIFY_CRON_SCHEDULE"
     fi
 
     # Add to crontab
     crontab /crontab.conf
 
-    echo "Starting restic cron..."
+    # List crontabs
-    cron
+    crontab -l
+
+    echo "Starting cron..."
+    crond
 
     touch /cron.log
     tail -f /cron.log

scripts/verify.sh

@@ -1,7 +1,8 @@
 #! /bin/bash
 set -e
 
+# shellcheck disable=SC2086
 restic \
-    -r $BACKUP_DEST \
+    -r "$BACKUP_DEST" \
     $OPT_ARGUMENTS \
     check

tests/docker-compose-test-s3.yml

@@ -1,9 +1,12 @@
+---
 version: '2'
 services:
   restic:
     build:
       context: ..
       dockerfile: Dockerfile
+      args:
+        - TARGETARCH
     entrypoint: "bash"
     command: "-c 'sleep 2 && /test.sh'"
     hostname: itest
@@ -26,5 +29,5 @@ services:
     expose:
       - "9000"
     environment:
-      MINIO_ACCESS_KEY: SUPER_SECRET_ACCESS_KEY
+      MINIO_ROOT_USER: SUPER_SECRET_ACCESS_KEY
-      MINIO_SECRET_KEY: SUPER_SECRET_SECRET_KEY
+      MINIO_ROOT_PASSWORD: SUPER_SECRET_SECRET_KEY

tests/test-pre-scripts.sh

@@ -1,7 +1,7 @@
 #! /bin/bash
 set -e
 
-image=$1
+image="$1"
 
 if [ "$IN_CONTAINER" != "true" ] ; then
     # Run the test script within the container
@@ -10,19 +10,20 @@ if [ "$IN_CONTAINER" != "true" ] ; then
         -e SKIP_ON_START=true \
         -e RESTIC_PASSWORD="Correct.Horse.Battery.Staple" \
         -v "$(pwd)/test-pre-scripts.sh:/test.sh" \
-        -v "$(pwd)/test-pre-scripts:/scripts" \
+        -v "$(pwd)/test-pre-scripts/backup:/scripts/backup" \
-        $image \
+        -v "$(pwd)/test-pre-scripts/restore:/scripts/restore" \
+        -v "$(pwd)/test-pre-scripts/create-test-data.sql:/scripts/create-test-data.sql" \
+        "$image" \
         bash -c "/test.sh"
 else
     echo "Performing backup tests"
 
     echo "Verify cron and crontab exist"
-    type cron
+    type crond
     type crontab
 
     echo "Install sqlite3"
-    apt-get update
+    apk add sqlite
-    apt-get install -y --no-install-recommends sqlite3
 
     echo "Create test data..."
     mkdir -p /data
@@ -30,10 +31,10 @@ else
     sqlite3 /data/test_database.db < /scripts/create-test-data.sql
 
     echo "Fake a start and init repo"
-    CRON_SCHEDULE="" /start.sh
+    CRON_SCHEDULE="" /scripts/start.sh
 
     echo "Making backup..."
-    /backup.sh
+    /scripts/backup.sh
 
     echo "Verify intermediary file is gone"
     test -f /data/test_database.db.bak && exit 1 || echo "Gone"
@@ -45,7 +46,7 @@ else
     test -f /data/test_database.db && exit 1 || echo "Gone"
 
     echo "Restore backup..."
-    /restore.sh latest
+    /scripts/restore.sh latest
 
     echo "Verify restored files exist..."
     test -f /data/test_database.db
@@ -59,7 +60,7 @@ else
     test -f /data/test_database.db && exit 1 || echo "Gone"
 
     echo "Simulate a restart with RESTORE_ON_EMPTY_START..."
-    RESTORE_ON_EMPTY_START=true /start.sh
+    RESTORE_ON_EMPTY_START=true /scripts/start.sh
 
     echo "Verify restore happened..."
     test -f /data/test_database.db
@@ -71,5 +72,5 @@ else
 
     echo "Verify restore with incorrect passphrase fails..."
     echo "Fail to restore backup..."
-    RESTIC_PASSWORD=Incorrect.Mule.Solar.Paperclip /restore.sh latest && exit 1 || echo "OK"
+    RESTIC_PASSWORD=Incorrect.Mule.Solar.Paperclip /scripts/restore.sh latest && exit 1 || echo "OK"
 fi

tests/test-pre-scripts/backup/after/post-backup.sh

@@ -1,3 +1,4 @@
+#! /bin/bash
 set -e
 
 cd /data

tests/test-pre-scripts/backup/before/pre-backup.sh

@@ -1,3 +1,4 @@
+#! /bin/bash
 set -e
 
 cd /data

tests/test-pre-scripts/restore/after/post-restore.sh

@@ -1,3 +1,4 @@
+#! /bin/bash
 set -e
 
 cd /data

tests/test-pre-scripts/restore/before/pre-restore.sh

@@ -1,3 +1,4 @@
+#! /bin/bash
 set -e
 
 # Don't really need to do anything here

tests/test-s3.sh

@@ -1,6 +1,4 @@
 #! /bin/bash
 
-export DOCKER_BASE=$1
-
 docker-compose -f docker-compose-test-s3.yml up \
     --build --abort-on-container-exit --force-recreate

tests/test.sh

@@ -1,7 +1,7 @@
 #! /bin/bash
 set -e
 
-image=$1
+image="$1"
 
 if [ "$IN_CONTAINER" != "true" ] ; then
     # Run the test script within the container
@@ -10,26 +10,29 @@ if [ "$IN_CONTAINER" != "true" ] ; then
         -e SKIP_ON_START=true \
         -e RESTIC_PASSWORD="Correct.Horse.Battery.Staple" \
         -v "$(pwd)/test.sh:/test.sh" \
-        $image \
+        "$image" \
         bash -c "/test.sh"
 else
     echo "Performing backup tests"
 
     echo "Verify cron and crontab exist"
-    type cron
+    type crond
     type crontab
 
     echo "Create test data..."
     mkdir -p /data && echo Test > /data/test.txt
 
     echo "Fake a start and init repo"
-    CRON_SCHEDULE="" /start.sh
+    CRON_SCHEDULE="" /scripts/start.sh
 
     echo "Making backup..."
-    /cron-exec.sh /backup.sh || { cat /cron.log && exit 1; }
+    /scripts/cron-exec.sh /scripts/backup.sh || { cat /cron.log && exit 1; }
 
     echo "Verify backup..."
-    /cron-exec.sh /verify.sh || { cat /cron.log && exit 1; }
+    /scripts/cron-exec.sh /scripts/verify.sh || { cat /cron.log && exit 1; }
+
+    echo "Auto cleanup on second backup..."
+    CLEANUP_COMMAND="--prune --keep-last 1" /scripts/cron-exec.sh /scripts/backup.sh || { cat /cron.log && exit 1; }
 
     echo "Delete test data..."
     rm -fr /data/*
@@ -38,15 +41,15 @@ else
     test -f /data/test.txt && exit 1 || echo "Gone"
 
     echo "Restore backup..."
-    /cron-exec.sh /restore.sh latest || { cat /cron.log && exit 1; }
+    /scripts/cron-exec.sh /scripts/restore.sh latest || { cat /cron.log && exit 1; }
-    /healthcheck.sh
+    /scripts/healthcheck.sh
 
     echo "Verify restore..."
     test -f /data/test.txt
     cat /data/test.txt
 
     echo "Verify backup..."
-    /verify.sh
+    /scripts/verify.sh
 
     echo "Delete test data again..."
     rm -fr /data/*
@@ -55,8 +58,8 @@ else
     test -f /data/test.txt && exit 1 || echo "Gone"
 
     echo "Simulate a restart with RESTORE_ON_EMPTY_START..."
-    RESTORE_ON_EMPTY_START=true /start.sh || { cat /cron.log && exit 1; }
+    RESTORE_ON_EMPTY_START=true /scripts/start.sh || { cat /cron.log && exit 1; }
-    /healthcheck.sh || { echo "Failed healthcheck"; cat /cron.log; exit 1; }
+    /scripts/healthcheck.sh || { echo "Failed healthcheck"; cat /cron.log; exit 1; }
 
     echo "Verify restore happened..."
     test -f /data/test.txt
@@ -64,8 +67,8 @@ else
 
     echo "Verify restore with incorrect passphrase fails..."
     echo "Fail to restore backup..."
-    RESTIC_PASSWORD=Incorrect.Mule.Solar.Paperclip /cron-exec.sh /restore.sh latest && exit 1 || echo "OK"
+    RESTIC_PASSWORD=Incorrect.Mule.Solar.Paperclip /scripts/cron-exec.sh /scripts/restore.sh latest && exit 1 || echo "OK"
 
     echo "Verify failed healthcheck"
-    /healthcheck.sh && exit 1 || echo "OK"
+    /scripts/healthcheck.sh && exit 1 || echo "OK"
 fi