Compare commits

..

36 Commits

Author SHA1 Message Date
IamTheFij 4b16dea34e Log when attempting to push metrics to gateway
continuous-integration/drone/push Build is passing Details
continuous-integration/drone/tag Build is passing Details
2024-01-23 12:26:10 -08:00
IamTheFij 7b313b8f9b Move log line to the proper location
continuous-integration/drone/push Build is passing Details
2024-01-22 08:50:49 -08:00
IamTheFij bea338c27a Add log when tasks are stopped
continuous-integration/drone/push Build is passing Details
2024-01-22 08:49:46 -08:00
IamTheFij 1d0d6b3fe6 More loosly pin tzdata
continuous-integration/drone/push Build is passing Details
continuous-integration/drone/tag Build is passing Details
2024-01-06 15:34:02 -08:00
IamTheFij 985572d737 Pin pre-commit plugin to specific sha
continuous-integration/drone/push Build is failing Details
continuous-integration/drone/tag Build is failing Details
2024-01-06 15:12:27 -08:00
IamTheFij cff06cd1c6 Update linters and fumpt all files
continuous-integration/drone/push Build was killed Details
2024-01-06 15:10:29 -08:00
IamTheFij 90cd0ec9e0 Update versions to fix golangci-lint on drone
continuous-integration/drone/push Build is failing Details
2024-01-06 14:55:44 -08:00
IamTheFij a0db27be1a Add ability to unlock repos that may have stale locks
continuous-integration/drone/push Build is failing Details
continuous-integration/drone/tag Build is failing Details
Defaults to remove all locks, even non-stale
2024-01-06 14:29:14 -08:00
IamTheFij cddc290ee0 Fix exhaustive structs 2024-01-06 14:28:52 -08:00
IamTheFij d049228980 Fix index out of range when reading snapshots list
continuous-integration/drone/push Build was killed Details
continuous-integration/drone/tag Build is passing Details
2023-11-07 11:03:18 -08:00
IamTheFij 390074e048 Add insecure-tls global option
continuous-integration/drone/push Build is passing Details
continuous-integration/drone/tag Build is passing Details
2023-11-06 15:02:07 -08:00
IamTheFij e0542a68e5 Add docker healthcheck checking default address
continuous-integration/drone/push Build is passing Details
2023-10-25 20:17:18 -07:00
IamTheFij 4ddcea9f7d Bump to alpine 3.18 and bump versions
continuous-integration/drone/push Build is passing Details
Also, more loosely pinning some dependencies with stable apis.
2023-10-25 20:16:26 -07:00
IamTheFij a2823e09ad Add tzdata
continuous-integration/drone/push Build is passing Details
Allows setting container timezone using TZ env variable
2023-08-11 05:53:17 -07:00
IamTheFij b1fe2537e2 Add postgres support for backup and restore
continuous-integration/drone/push Build is passing Details
continuous-integration/drone/tag Build is passing Details
2023-08-02 15:58:41 -07:00
IamTheFij f3ecabf4fe Fix mysql test
continuous-integration/drone/push Build is passing Details
continuous-integration/drone/tag Build is passing Details
2023-08-02 15:56:06 -07:00
IamTheFij 4c7baa46a7 Ignore binary
continuous-integration/drone/push Build is passing Details
2023-08-02 15:41:15 -07:00
IamTheFij 1d6957f45f Fix mysql restoration
Password and database weren't passed to the mysql command
2023-08-02 15:41:15 -07:00
IamTheFij 0de267a4cf Disable depguard 2023-08-02 14:58:18 -07:00
IamTheFij 28f081c8d0 Add integration testing to verify backup and restoration
continuous-integration/drone/push Build is passing Details
Including databases
2023-08-02 14:55:14 -07:00
IamTheFij 9cdf37c680 Fix wrapped snapshot json output
continuous-integration/drone/push Build is passing Details
continuous-integration/drone/tag Build is passing Details
If there are a lot of snapshots, the JSON output can sometimes be wrapped to
multiple console lines and need to be joined.
2023-05-09 15:10:44 -07:00
IamTheFij 84095f9875 Remove patch version from Nomad binary
continuous-integration/drone/push Build is passing Details
continuous-integration/drone/tag Build is passing Details
2023-05-09 14:31:44 -07:00
IamTheFij 03cc5e8fa6 Remove darwin binaries from build targets 2023-05-09 14:30:04 -07:00
IamTheFij 129dc75d4c Remove failure for coverage
continuous-integration/drone/push Build is failing Details
This seems to be broken on newer versions of Golang
2023-05-09 14:23:22 -07:00
IamTheFij f32b5e4afa Bump go version up to 1.20
continuous-integration/drone/push Build is failing Details
2023-05-09 14:19:21 -07:00
IamTheFij a19ad94027 Serve metrics from registry
continuous-integration/drone/push Build is failing Details
2023-05-09 14:16:39 -07:00
IamTheFij 8b9844465c Clean up error messages 2023-05-09 14:16:08 -07:00
IamTheFij 95fea5ef30 Add ability to restore specific snapshots
continuous-integration/drone/push Build is failing Details
2023-04-25 14:02:28 -07:00
IamTheFij 8a8bc23376 Split returned stdout and stderr from Restic commands to improve parsing 2023-04-25 13:59:32 -07:00
IamTheFij fed9224c17 Add some additional test run output 2023-04-25 13:32:37 -07:00
IamTheFij 47f1c14c48 Bump alpine and dep versions and add nomad
continuous-integration/drone/push Build is passing Details
2023-04-04 16:48:26 -07:00
IamTheFij bd3151e6f8 Add support for prometheus push gateway on single runs 2023-01-06 15:38:05 -08:00
IamTheFij 3e550d29ca Add notes for future 2023-01-06 15:37:31 -08:00
IamTheFij 0074cced26 Pin Alpine version to 3.16
continuous-integration/drone/push Build is passing Details
Package versions always break in builds when a new Alpine version comes
out. Im going to start pinning these.
2022-11-27 21:56:01 -08:00
IamTheFij 9c5d597ab4 Add DefaultConfig
continuous-integration/drone/push Build is failing Details
Adds DefaultConfig to the config format so that a single config can be
defined for all jobs
2022-11-27 21:05:02 -08:00
IamTheFij 57e933c20d Fix error when passing no names
continuous-integration/drone/push Build is passing Details
2022-11-16 09:52:29 -08:00
23 changed files with 738 additions and 137 deletions

View File

@ -4,7 +4,7 @@ name: test
steps:
- name: test
image: golang:1.17
image: golang:1.21
environment:
VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
commands:
@ -13,7 +13,7 @@ steps:
- make test
- name: check
image: iamthefij/drone-pre-commit:personal
image: iamthefij/drone-pre-commit@sha256:30fa17489b86d7a4c3ad9c3ce2e152c25d82b8671e5609d322c6cae0baed89cd
---
kind: pipeline
@ -32,7 +32,7 @@ trigger:
steps:
- name: build all binaries
image: golang:1.17
image: golang:1.21
environment:
VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
commands:

5
.gitignore vendored
View File

@ -20,4 +20,9 @@ dist/
# Built executable
restic-scheduler
resticscheduler
data/
# Itest temp dirs
itest/data
itest/repo

View File

@ -16,7 +16,6 @@ linters:
- contextcheck
- cyclop
- decorder
- depguard
- dupl
- durationcheck
- errchkjson
@ -32,10 +31,8 @@ linters:
- gocognit
- goconst
- gocritic
# - gocyclo # Using cyclop
- godot
# - goerr113 # Using errorlint
- gofmt
- gofumpt
- goheader
- goimports
- gomnd
@ -45,7 +42,6 @@ linters:
- gosec
- grouper
- importas
# - ireturn
- lll
- maintidx
- makezero
@ -60,11 +56,9 @@ linters:
- paralleltest
- prealloc
- predeclared
# - promlinter # Not common enough
- revive
- rowserrcheck
- sqlclosecheck
# - stylecheck # Using revive
- tagliatelle
- tenv
- testpackage
@ -72,27 +66,11 @@ linters:
- tparallel
- unconvert
- unparam
- varnamelen
- wastedassign
- whitespace
- wrapcheck
- wsl
disable:
- gochecknoglobals
- godox
- forbidigo
# Deprecated
- golint
- interfacer
- maligned
- scopelint
- ifshort
- varcheck
- structcheck
- deadcode
- exhaustivestruct
linters-settings:
gomnd:
settings:

View File

@ -1,7 +1,7 @@
---
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.4.0
rev: v4.5.0
hooks:
- id: check-added-large-files
- id: check-yaml
@ -11,10 +11,8 @@ repos:
- id: end-of-file-fixer
- id: check-merge-conflict
- repo: https://github.com/dnephin/pre-commit-golang
rev: v0.4.0
rev: v0.5.1
hooks:
- id: go-fmt
- id: go-imports
- id: golangci-lint
args:
- --timeout=3m

View File

@ -1,18 +1,23 @@
FROM alpine:3.16
FROM alpine:3.18
RUN apk add --no-cache \
bash~=5 \
consul~=1.12 \
mariadb-client~=10.6 \
mariadb-connector-c~=3.1 \
rclone~=1.58 \
redis~=7.0 \
restic~=0.13 \
consul~=1 \
mariadb-client~=10 \
mariadb-connector-c~=3 \
nomad~=1 \
postgresql15-client~=15 \
rclone~=1.62 \
redis~=7 \
restic~=0.15 \
sqlite~=3 \
tzdata~=2023 \
;
ARG TARGETOS
ARG TARGETARCH
COPY ./dist/resticscheduler-$TARGETOS-$TARGETARCH /bin/resticscheduler
HEALTHCHECK CMD ["wget", "-O", "-", "http://localhost:8080/health"]
ENTRYPOINT [ "/bin/resticscheduler" ]

View File

@ -2,7 +2,7 @@ APP_NAME = resticscheduler
VERSION ?= $(shell git describe --tags --dirty)
GOFILES = *.go
# Multi-arch targets are generated from this
TARGET_ALIAS = $(APP_NAME)-linux-amd64 $(APP_NAME)-linux-arm $(APP_NAME)-linux-arm64 $(APP_NAME)-darwin-amd64 $(APP_NAME)-darwin-arm64
TARGET_ALIAS = $(APP_NAME)-linux-amd64 $(APP_NAME)-linux-arm $(APP_NAME)-linux-arm64
TARGETS = $(addprefix dist/,$(TARGET_ALIAS))
.QUOTE = "
CURRENT_GOARCH = $(shell go env | awk -F "=" '/GOARCH/ { gsub(/$(.QUOTE)/,"", $$2); print $$2}')
@ -29,9 +29,13 @@ build: $(APP_NAME)
# Run all tests
.PHONY: test
test:
go test -coverprofile=coverage.out # -short
go test -v -coverprofile=coverage.out # -short
go tool cover -func=coverage.out
.PHONY: itest
itest: docker-build
./itest/run.sh
# Installs pre-commit hooks
.PHONY: install-hooks
install-hooks:

2
go.mod
View File

@ -1,6 +1,6 @@
module git.iamthefij.com/iamthefij/restic-scheduler
go 1.17
go 1.20
require (
github.com/go-test/deep v1.0.8

46
itest/bootstrap-tests.sh Executable file
View File

@ -0,0 +1,46 @@
#! /bin/sh
set -ex
# Create flat file
echo "Hello" > /data/test.txt
# Create Sqlite database
touch /data/test_database.db
sqlite3 /data/test_database.db <<-EOF
CREATE TABLE test_table (
id INTEGER PRIMARY KEY,
data TEXT NOT NULL
);
INSERT INTO test_table(data)
VALUES ("Test row");
EOF
# Create MySql database
until mysql --host "$MYSQL_HOST" --user "$MYSQL_USER" --password="$MYSQL_PWD" --execute "SHOW DATABASES;"; do
sleep 1
done
mysql --host "$MYSQL_HOST" --user "$MYSQL_USER" --password="$MYSQL_PWD" main <<EOF
CREATE TABLE test_table (
id INTEGER AUTO_INCREMENT PRIMARY KEY,
data TEXT NOT NULL
);
INSERT INTO test_table(data)
VALUES ("Test row");
EOF
# Create Postgres database
export PGPASSWORD="$PGSQL_PASS"
until psql --host "$PGSQL_HOST" --username "$PGSQL_USER" --command "SELECT datname FROM pg_database;"; do
sleep 1
done
psql -v ON_ERROR_STOP=1 --host "$PGSQL_HOST" --username "$PGSQL_USER" main <<EOF
CREATE TABLE test_table (
id SERIAL PRIMARY KEY,
data TEXT NOT NULL
);
INSERT INTO test_table(data)
VALUES ('Test row');
EOF

57
itest/docker-compose.yml Normal file
View File

@ -0,0 +1,57 @@
---
version: "3.9"
services:
mysql:
image: mysql
environment:
MYSQL_ROOT_PASSWORD: shhh
MYSQL_DATABASE: main
postgres:
image: postgres
environment:
POSTGRES_PASSWORD: shhh
POSTGRES_DB: main
bootstrap:
image: resticscheduler
entrypoint: /bootstrap-tests.sh
environment:
MYSQL_HOST: mysql
MYSQL_USER: root
MYSQL_PWD: shhh
PGSQL_HOST: postgres
PGSQL_USER: postgres
PGSQL_PASS: shhh
volumes:
- ./bootstrap-tests.sh:/bootstrap-tests.sh
- ./data:/data
main:
image: resticscheduler
environment:
MYSQL_HOST: mysql
MYSQL_USER: root
MYSQL_PWD: shhh
PGSQL_HOST: postgres
PGSQL_USER: postgres
PGSQL_PASS: shhh
volumes:
- ./repo:/repo
- ./data:/data
- ./test-backup.hcl:/test-backup.hcl
validate:
image: resticscheduler
entrypoint: /validate-tests.sh
environment:
MYSQL_HOST: mysql
MYSQL_USER: root
MYSQL_PWD: shhh
PGSQL_HOST: postgres
PGSQL_USER: postgres
PGSQL_PASS: shhh
volumes:
- ./validate-tests.sh:/validate-tests.sh
- ./data:/data

35
itest/run.sh Executable file
View File

@ -0,0 +1,35 @@
#! /bin/bash
set -ex
cd "$(dirname "$0")"
mkdir -p ./repo ./data
echo Clean everything
docker-compose down -v
rm -fr ./repo/* ./data/*
sleep 5
echo Boostrap databases and data
docker-compose up -d mysql postgres
docker-compose run bootstrap
sleep 1
echo Run backup job
docker-compose run main -backup IntegrationTest -once /test-backup.hcl
echo Clean data
docker-compose down -v
docker-compose up -d mysql postgres
rm -fr ./data/*
sleep 15
echo Run restore
docker-compose run main -restore IntegrationTest -once /test-backup.hcl
sleep 1
echo Validate data
docker-compose run validate
echo Clean all again
docker-compose down -v
rm -fr ./repo/* ./data/*

38
itest/test-backup.hcl Normal file
View File

@ -0,0 +1,38 @@
job "IntegrationTest" {
schedule = "@daily"
config {
repo = "/repo"
passphrase = "shh"
}
mysql "MySQL" {
hostname = env("MYSQL_HOST")
database = "main"
username = env("MYSQL_USER")
password = env("MYSQL_PWD")
dump_to = "/tmp/mysql.sql"
}
postgres "Postgres" {
hostname = env("PGSQL_HOST")
database = "main"
username = env("PGSQL_USER")
password = env("PGSQL_PASS")
create = true
dump_to = "/tmp/psql.sql"
}
sqlite "SQLite" {
path = "/data/test_database.db"
dump_to = "/data/test_database.db.bak"
}
backup {
paths = ["/data"]
restore_opts {
Target = "/"
}
}
}

21
itest/validate-tests.sh Executable file
View File

@ -0,0 +1,21 @@
#! /bin/sh
set -ex
# Check flat file
test -f /data/test.txt
grep "^Hello" /data/test.txt
# Check Sqlite database
test -f /data/test_database.db
sqlite3 /data/test_database.db "select data from test_table where id = 1" | grep "^Test row"
# Check MySql database
mysql --host "$MYSQL_HOST" --user "$MYSQL_USER" --password="$MYSQL_PWD" main <<EOF | grep "^Test row"
select data from test_table where id = 1;
EOF
# Check Postgres database
export PGPASSWORD="$PGSQL_PASS"
psql --host "$PGSQL_HOST" --user "$PGSQL_USER" main <<EOF | grep "Test row"
select data from test_table where id = 1;
EOF

79
job.go
View File

@ -52,14 +52,17 @@ func (r ResticConfig) Validate() error {
type Job struct {
Name string `hcl:"name,label"`
Schedule string `hcl:"schedule"`
Config ResticConfig `hcl:"config,block"`
Config *ResticConfig `hcl:"config,block"`
Tasks []JobTask `hcl:"task,block"`
Backup BackupFilesTask `hcl:"backup,block"`
Forget *ForgetOpts `hcl:"forget,block"`
// Meta Tasks
MySQL []JobTaskMySQL `hcl:"mysql,block"`
Sqlite []JobTaskSqlite `hcl:"sqlite,block"`
// NOTE: Now that these are also available within a task
// these could be removed to make task order more obvious
MySQL []JobTaskMySQL `hcl:"mysql,block"`
Postgres []JobTaskPostgres `hcl:"postgres,block"`
Sqlite []JobTaskSqlite `hcl:"sqlite,block"`
// Metrics and health
healthy bool
@ -73,6 +76,24 @@ func (j Job) validateTasks() error {
}
}
for _, mysql := range j.MySQL {
if err := mysql.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
for _, pg := range j.Postgres {
if err := pg.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
for _, sqlite := range j.Sqlite {
if err := sqlite.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
return nil
}
@ -82,7 +103,11 @@ func (j Job) Validate() error {
}
if _, err := cron.ParseStandard(j.Schedule); err != nil {
return fmt.Errorf("job %s has an invalid schedule: %v: %w", j.Name, err, ErrInvalidConfigValue)
return fmt.Errorf("job %s has an invalid schedule: %w: %w", j.Name, err, ErrInvalidConfigValue)
}
if j.Config == nil {
return fmt.Errorf("job %s is missing restic config: %w", j.Name, ErrMissingField)
}
if err := j.Config.Validate(); err != nil {
@ -93,18 +118,6 @@ func (j Job) Validate() error {
return err
}
for _, mysql := range j.MySQL {
if err := mysql.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
for _, sqlite := range j.Sqlite {
if err := sqlite.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
if err := j.Backup.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid backup config: %w", j.Name, err)
}
@ -120,6 +133,10 @@ func (j Job) AllTasks() []ExecutableTask {
allTasks = append(allTasks, mysql.GetPreTask())
}
for _, pg := range j.Postgres {
allTasks = append(allTasks, pg.GetPreTask())
}
for _, sqlite := range j.Sqlite {
allTasks = append(allTasks, sqlite.GetPreTask())
}
@ -140,6 +157,10 @@ func (j Job) AllTasks() []ExecutableTask {
allTasks = append(allTasks, mysql.GetPostTask())
}
for _, pg := range j.Postgres {
allTasks = append(allTasks, pg.GetPostTask())
}
for _, sqlite := range j.Sqlite {
allTasks = append(allTasks, sqlite.GetPostTask())
}
@ -154,6 +175,10 @@ func (j Job) BackupPaths() []string {
paths = append(paths, t.DumpToPath)
}
for _, t := range j.Postgres {
paths = append(paths, t.DumpToPath)
}
for _, t := range j.Sqlite {
paths = append(paths, t.DumpToPath)
}
@ -197,7 +222,7 @@ func (j Job) Logger() *log.Logger {
return GetLogger(j.Name)
}
func (j Job) RunRestore() error {
func (j Job) RunRestore(snapshot string) error {
logger := j.Logger()
restic := j.NewRestic()
@ -213,6 +238,10 @@ func (j Job) RunRestore() error {
Env: nil,
}
if backupTask, ok := exTask.(BackupFilesTask); ok {
backupTask.snapshot = snapshot
}
if err := exTask.RunRestore(taskCfg); err != nil {
return fmt.Errorf("failed running job %s: %w", j.Name, err)
}
@ -251,8 +280,10 @@ func (j Job) Run() {
result.LastError = err
} else {
Metrics.SnapshotCurrentCount.WithLabelValues(j.Name).Set(float64(len(snapshots)))
latestSnapshot := snapshots[len(snapshots)-1]
Metrics.SnapshotLatestTime.WithLabelValues(j.Name).Set(float64(latestSnapshot.Time.Unix()))
if len(snapshots) > 0 {
latestSnapshot := snapshots[len(snapshots)-1]
Metrics.SnapshotLatestTime.WithLabelValues(j.Name).Set(float64(latestSnapshot.Time.Unix()))
}
}
if result.Success {
@ -276,8 +307,8 @@ func (j Job) NewRestic() *Restic {
}
type Config struct {
// GlobalConfig *ResticConfig `hcl:"global_config,block"`
Jobs []Job `hcl:"job,block"`
DefaultConfig *ResticConfig `hcl:"default_config,block"`
Jobs []Job `hcl:"job,block"`
}
func (c Config) Validate() error {
@ -286,6 +317,12 @@ func (c Config) Validate() error {
}
for _, job := range c.Jobs {
// Use default restic config if no job config is provided
// TODO: Maybe merge values here
if job.Config == nil {
job.Config = c.DefaultConfig
}
if err := job.Validate(); err != nil {
return err
}

View File

@ -7,8 +7,8 @@ import (
main "git.iamthefij.com/iamthefij/restic-scheduler"
)
func ValidResticConfig() main.ResticConfig {
return main.ResticConfig{
func ValidResticConfig() *main.ResticConfig {
return &main.ResticConfig{
Passphrase: "shh",
Repo: "./data",
Env: nil,
@ -92,6 +92,7 @@ func TestJobValidation(t *testing.T) {
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
},
expectedErr: nil,
@ -106,6 +107,7 @@ func TestJobValidation(t *testing.T) {
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
},
expectedErr: main.ErrMissingField,
@ -120,6 +122,7 @@ func TestJobValidation(t *testing.T) {
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
},
expectedErr: main.ErrInvalidConfigValue,
@ -129,11 +132,12 @@ func TestJobValidation(t *testing.T) {
job: main.Job{
Name: "Test job",
Schedule: "@daily",
Config: main.ResticConfig{}, //nolint:exhaustruct
Config: &main.ResticConfig{}, //nolint:exhaustruct
Tasks: []main.JobTask{},
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
},
expectedErr: main.ErrMutuallyExclusive,
@ -144,10 +148,13 @@ func TestJobValidation(t *testing.T) {
Name: "Test job",
Schedule: "@daily",
Config: ValidResticConfig(),
Tasks: []main.JobTask{{}},
Tasks: []main.JobTask{
{}, //nolint:exhaustruct
},
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
},
expectedErr: main.ErrMissingField,
@ -161,7 +168,10 @@ func TestJobValidation(t *testing.T) {
Tasks: []main.JobTask{},
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{{}},
MySQL: []main.JobTaskMySQL{
{}, //nolint:exhaustruct
},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
},
expectedErr: main.ErrMissingField,
@ -176,7 +186,10 @@ func TestJobValidation(t *testing.T) {
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Sqlite: []main.JobTaskSqlite{{}},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{
{}, //nolint:exhaustruct
},
},
expectedErr: main.ErrMissingField,
},
@ -207,34 +220,82 @@ func TestConfigValidation(t *testing.T) {
}{
{
name: "Valid job",
config: main.Config{Jobs: []main.Job{{
Name: "Valid job",
Schedule: "@daily",
Config: ValidResticConfig(),
Tasks: []main.JobTask{},
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
MySQL: []main.JobTaskMySQL{},
Sqlite: []main.JobTaskSqlite{},
}}},
config: main.Config{
DefaultConfig: nil,
Jobs: []main.Job{{
Name: "Valid job",
Schedule: "@daily",
Config: ValidResticConfig(),
Tasks: []main.JobTask{},
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
}},
},
expectedErr: nil,
},
{
name: "No jobs",
config: main.Config{Jobs: []main.Job{}},
name: "Valid job with default config",
config: main.Config{
DefaultConfig: ValidResticConfig(),
Jobs: []main.Job{{
Name: "Valid job",
Schedule: "@daily",
Config: nil,
Tasks: []main.JobTask{},
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
}},
},
expectedErr: nil,
},
{
name: "No jobs",
config: main.Config{
DefaultConfig: nil,
Jobs: []main.Job{},
},
expectedErr: main.ErrNoJobsFound,
},
{
name: "Invalid name",
config: main.Config{Jobs: []main.Job{{
Name: "",
Schedule: "@daily",
Config: ValidResticConfig(),
Tasks: []main.JobTask{},
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Sqlite: []main.JobTaskSqlite{},
}}},
config: main.Config{
DefaultConfig: nil,
Jobs: []main.Job{{
Name: "",
Schedule: "@daily",
Config: ValidResticConfig(),
Tasks: []main.JobTask{},
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
}},
},
expectedErr: main.ErrMissingField,
},
{
name: "Missing config",
config: main.Config{
DefaultConfig: nil,
Jobs: []main.Job{{
Name: "",
Schedule: "@daily",
Config: nil,
Tasks: []main.JobTask{},
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
}},
},
expectedErr: main.ErrMissingField,
},
}

66
main.go
View File

@ -29,8 +29,12 @@ func ParseConfig(path string) ([]Job, error) {
Functions: map[string]function.Function{
"env": function.New(&function.Spec{
Params: []function.Parameter{{
Name: "var",
Type: cty.String,
Name: "var",
Type: cty.String,
AllowNull: false,
AllowUnknown: false,
AllowDynamicType: false,
AllowMarked: false,
}},
VarParam: nil,
Type: function.StaticReturnType(cty.String),
@ -40,8 +44,12 @@ func ParseConfig(path string) ([]Job, error) {
}),
"readfile": function.New(&function.Spec{
Params: []function.Parameter{{
Name: "path",
Type: cty.String,
Name: "path",
Type: cty.String,
AllowNull: false,
AllowUnknown: false,
AllowDynamicType: false,
AllowMarked: false,
}},
VarParam: nil,
Type: function.StaticReturnType(cty.String),
@ -140,6 +148,10 @@ func FilterJobs(jobs []Job, names []string) ([]Job, error) {
}
func runBackupJobs(jobs []Job, names string) error {
if names == "" {
return nil
}
namesSlice := strings.Split(names, ",")
if len(namesSlice) == 0 {
@ -156,7 +168,11 @@ func runBackupJobs(jobs []Job, names string) error {
return filterJobErr
}
func runRestoreJobs(jobs []Job, names string) error {
func runRestoreJobs(jobs []Job, names string, snapshot string) error {
if names == "" {
return nil
}
namesSlice := strings.Split(names, ",")
if len(namesSlice) == 0 {
@ -165,7 +181,28 @@ func runRestoreJobs(jobs []Job, names string) error {
jobs, filterJobErr := FilterJobs(jobs, namesSlice)
for _, job := range jobs {
if err := job.RunRestore(); err != nil {
if err := job.RunRestore(snapshot); err != nil {
return err
}
}
return filterJobErr
}
func runUnlockJobs(jobs []Job, names string) error {
if names == "" {
return nil
}
namesSlice := strings.Split(names, ",")
if len(namesSlice) == 0 {
return nil
}
jobs, filterJobErr := FilterJobs(jobs, namesSlice)
for _, job := range jobs {
if err := job.NewRestic().Unlock(UnlockOpts{RemoveAll: true}); err != nil {
return err
}
}
@ -177,6 +214,8 @@ type Flags struct {
showVersion bool
backup string
restore string
unlock string
restoreSnapshot string
once bool
healthCheckAddr string
metricsPushGateway string
@ -187,23 +226,30 @@ func readFlags() Flags {
flag.BoolVar(&flags.showVersion, "version", false, "Display the version and exit")
flag.StringVar(&flags.backup, "backup", "", "Run backup jobs now. Names are comma separated. `all` will run all.")
flag.StringVar(&flags.restore, "restore", "", "Run restore jobs now. Names are comma separated. `all` will run all.")
flag.StringVar(&flags.unlock, "unlock", "", "Unlock job repos now. Names are comma separated. `all` will run all.")
flag.BoolVar(&flags.once, "once", false, "Run jobs specified using -backup and -restore once and exit")
flag.StringVar(&flags.healthCheckAddr, "addr", "0.0.0.0:8080", "address to bind health check API")
flag.StringVar(&flags.metricsPushGateway, "push-gateway", "", "url of push gateway service for batch runs (optional)")
flag.StringVar(&JobBaseDir, "base-dir", JobBaseDir, "Base dir to create intermediate job files like SQL dumps.")
flag.StringVar(&flags.restoreSnapshot, "snapshot", "latest", "the snapshot to restore")
flag.Parse()
return flags
}
func runSpecifiedJobs(jobs []Job, backupJobs, restoreJobs string) error {
func runSpecifiedJobs(jobs []Job, backupJobs, restoreJobs, unlockJobs, snapshot string) error {
// Run specified job unlocks
if err := runUnlockJobs(jobs, unlockJobs); err != nil {
return fmt.Errorf("Failed running unlock for jobs: %w", err)
}
// Run specified backup jobs
if err := runBackupJobs(jobs, backupJobs); err != nil {
return fmt.Errorf("Failed running backup jobs: %w", err)
}
// Run specified restore jobs
if err := runRestoreJobs(jobs, restoreJobs); err != nil {
if err := runRestoreJobs(jobs, restoreJobs, snapshot); err != nil {
return fmt.Errorf("Failed running restore jobs: %w", err)
}
@ -212,6 +258,8 @@ func runSpecifiedJobs(jobs []Job, backupJobs, restoreJobs string) error {
func maybePushMetrics(metricsPushGateway string) error {
if metricsPushGateway != "" {
fmt.Println("Pushing metrics to push gateway")
if err := Metrics.PushToGateway(metricsPushGateway); err != nil {
return fmt.Errorf("Failed pushing metrics after jobs run: %w", err)
}
@ -243,7 +291,7 @@ func main() {
log.Fatalf("Failed to read jobs from files: %v", err)
}
if err := runSpecifiedJobs(jobs, flags.backup, flags.restore); err != nil {
if err := runSpecifiedJobs(jobs, flags.backup, flags.restore, flags.unlock, flags.restoreSnapshot); err != nil {
log.Fatal(err)
}

View File

@ -18,9 +18,9 @@ func TestMain(m *testing.M) {
if testResult == 0 && testing.CoverMode() != "" {
c := testing.Coverage()
if c < MinCoverage {
fmt.Printf("Tests passed but coverage failed at %0.2f and minimum to pass is %0.2f\n", c, MinCoverage)
fmt.Printf("WARNING: Tests passed but coverage failed at %0.2f and minimum to pass is %0.2f\n", c, MinCoverage)
testResult = -1
testResult = 0
}
}
@ -31,7 +31,6 @@ func TestReadJobs(t *testing.T) {
t.Parallel()
jobs, err := main.ReadJobs([]string{"./test/sample.hcl"})
if err != nil {
t.Errorf("Unexpected error reading jobs: %v", err)
}
@ -52,6 +51,7 @@ func TestRunJobs(t *testing.T) {
Backup: main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
Forget: nil,
MySQL: []main.JobTaskMySQL{},
Postgres: []main.JobTaskPostgres{},
Sqlite: []main.JobTaskSqlite{},
}

View File

@ -19,7 +19,6 @@ func (m ResticMetrics) PushToGateway(url string) error {
err := push.New(url, "batch").
Gatherer(m.Registry).
Add()
if err != nil {
return fmt.Errorf("error pushing to registry %s: %w", url, err)
}

View File

@ -11,8 +11,10 @@ import (
"time"
)
var ErrRestic = errors.New("restic error")
var ErrRepoNotFound = errors.New("repository not found or uninitialized")
var (
ErrRestic = errors.New("restic error")
ErrRepoNotFound = errors.New("repository not found or uninitialized")
)
func lineIn(needle string, haystack []string) bool {
for _, line := range haystack {
@ -72,6 +74,16 @@ func (NoOpts) ToArgs() []string {
return []string{}
}
type UnlockOpts struct {
RemoveAll bool `hcl:"RemoveAll,optional"`
}
func (uo UnlockOpts) ToArgs() (args []string) {
args = maybeAddArgBool(args, "--remove-all", uo.RemoveAll)
return
}
type BackupOpts struct {
Exclude []string `hcl:"Exclude,optional"`
Include []string `hcl:"Include,optional"`
@ -193,9 +205,10 @@ type ResticGlobalOpts struct {
TLSClientCertFile string `hcl:"TlsClientCertFile,optional"`
LimitDownload int `hcl:"LimitDownload,optional"`
LimitUpload int `hcl:"LimitUpload,optional"`
Options map[string]string `hcl:"Options,optional"`
VerboseLevel int `hcl:"VerboseLevel,optional"`
Options map[string]string `hcl:"Options,optional"`
CleanupCache bool `hcl:"CleanupCache,optional"`
InsecureTLS bool `hcl:"InsecureTls,optional"`
NoCache bool `hcl:"NoCache,optional"`
NoLock bool `hcl:"NoLock,optional"`
}
@ -209,6 +222,7 @@ func (glo ResticGlobalOpts) ToArgs() (args []string) {
args = maybeAddArgInt(args, "--limit-upload", glo.LimitUpload)
args = maybeAddArgInt(args, "--verbose", glo.VerboseLevel)
args = maybeAddArgBool(args, "--cleanup-cache", glo.CleanupCache)
args = maybeAddArgBool(args, "--insecure-tls", glo.InsecureTLS)
args = maybeAddArgBool(args, "--no-cache", glo.NoCache)
args = maybeAddArgBool(args, "--no-lock", glo.NoLock)
@ -273,7 +287,11 @@ func (e *ResticError) Unwrap() error {
return e.OriginalError
}
func (rcmd Restic) RunRestic(command string, options CommandOptions, commandArgs ...string) ([]string, error) {
func (rcmd Restic) RunRestic(
command string,
options CommandOptions,
commandArgs ...string,
) (*CapturedCommandLogWriter, error) {
args := []string{}
if rcmd.GlobalOpts != nil {
args = rcmd.GlobalOpts.ToArgs()
@ -285,22 +303,22 @@ func (rcmd Restic) RunRestic(command string, options CommandOptions, commandArgs
cmd := exec.Command("restic", args...)
output := NewCapturedLogWriter(rcmd.Logger)
cmd.Stdout = output
cmd.Stderr = output
output := NewCapturedCommandLogWriter(rcmd.Logger)
cmd.Stdout = output.Stdout
cmd.Stderr = output.Stderr
cmd.Env = rcmd.BuildEnv()
cmd.Dir = rcmd.Cwd
if err := cmd.Run(); err != nil {
responseErr := ErrRestic
if lineIn("Is there a repository at the following location?", output.Lines) {
if lineIn("Is there a repository at the following location?", output.Stderr.Lines) {
responseErr = ErrRepoNotFound
}
return output.Lines, NewResticError(command, output.Lines, responseErr)
return output, NewResticError(command, output.AllLines(), responseErr)
}
return output.Lines, nil
return output, nil
}
func (rcmd Restic) Backup(files []string, opts BackupOpts) error {
@ -327,6 +345,12 @@ func (rcmd Restic) Check() error {
return err
}
func (rcmd Restic) Unlock(unlockOpts UnlockOpts) error {
_, err := rcmd.RunRestic("unlock", unlockOpts)
return err
}
type Snapshot struct {
UID int `json:"uid"`
GID int `json:"gid"`
@ -341,15 +365,20 @@ type Snapshot struct {
}
func (rcmd Restic) ReadSnapshots() ([]Snapshot, error) {
lines, err := rcmd.RunRestic("snapshots", GenericOpts{"--json"})
output, err := rcmd.RunRestic("snapshots", GenericOpts{"--json"})
if err != nil {
return nil, err
}
snapshots := new([]Snapshot)
if len(output.Stdout.Lines) == 0 {
return nil, fmt.Errorf("no snapshot output to parse: %w", ErrRestic)
}
if err = json.Unmarshal([]byte(lines[0]), snapshots); err != nil {
return nil, fmt.Errorf("failed parsing snapshot results from %s: %w", lines[0], err)
singleLineOutput := strings.Join(output.Stdout.Lines, "")
snapshots := new([]Snapshot)
if err = json.Unmarshal([]byte(singleLineOutput), snapshots); err != nil {
return nil, fmt.Errorf("failed parsing snapshot results from %s: %w", singleLineOutput, err)
}
return *snapshots, nil

View File

@ -32,6 +32,7 @@ func TestGlobalOptions(t *testing.T) {
LimitUpload: 1,
VerboseLevel: 1,
CleanupCache: true,
InsecureTLS: true,
NoCache: true,
NoLock: true,
Options: map[string]string{
@ -48,6 +49,7 @@ func TestGlobalOptions(t *testing.T) {
"--limit-upload", "1",
"--verbose", "1",
"--cleanup-cache",
"--insecure-tls",
"--no-cache",
"--no-lock",
"--option", "key='a long value'",
@ -150,6 +152,20 @@ func TestForgetOpts(t *testing.T) {
AssertEqual(t, "args didn't match", expected, args)
}
func TestUnlockOpts(t *testing.T) {
t.Parallel()
args := main.UnlockOpts{
RemoveAll: true,
}.ToArgs()
expected := []string{
"--remove-all",
}
AssertEqual(t, "args didn't match", expected, args)
}
func TestBuildEnv(t *testing.T) {
t.Parallel()
@ -221,7 +237,7 @@ func TestResticInterface(t *testing.T) {
}
// Write test file to the data dir
err := os.WriteFile(dataFile, []byte("testing"), 0644)
err := os.WriteFile(dataFile, []byte("testing"), 0o644)
AssertEqualFail(t, "unexpected error writing to test file", nil, err)
// Make sure no existing repo is found
@ -281,7 +297,7 @@ func TestResticInterface(t *testing.T) {
AssertEqualFail(t, "unexpected error checking repo", nil, err)
// Change the data file
err = os.WriteFile(dataFile, []byte("unexpected"), 0644)
err = os.WriteFile(dataFile, []byte("unexpected"), 0o644)
AssertEqualFail(t, "unexpected error writing to test file", nil, err)
// Check that data wrote
@ -297,4 +313,8 @@ func TestResticInterface(t *testing.T) {
value, err = os.ReadFile(restoredDataFile)
AssertEqualFail(t, "unexpected error reading from test file", nil, err)
AssertEqualFail(t, "incorrect value in test file", "testing", string(value))
// Try to unlock the repo (repo shouldn't really be locked, but this should still run without error
err = restic.Unlock(main.UnlockOpts{}) //nolint:exhaustruct
AssertEqualFail(t, "unexpected error unlocking repo", nil, err)
}

View File

@ -13,8 +13,10 @@ import (
"github.com/robfig/cron/v3"
)
var jobResultsLock = sync.Mutex{}
var jobResults = map[string]JobResult{}
var (
jobResultsLock = sync.Mutex{}
jobResults = map[string]JobResult{}
)
type JobResult struct {
JobName string
@ -67,9 +69,12 @@ func healthHandleFunc(writer http.ResponseWriter, request *http.Request) {
func RunHTTPHandlers(addr string) error {
http.HandleFunc("/health", healthHandleFunc)
http.Handle("/metrics", promhttp.Handler())
http.Handle("/metrics", promhttp.HandlerFor(
Metrics.Registry,
promhttp.HandlerOpts{Registry: Metrics.Registry}, //nolint:exhaustruct
))
return fmt.Errorf("error on healthcheck: %w", http.ListenAndServe(addr, nil)) //#nosec: g114
return fmt.Errorf("error on http server: %w", http.ListenAndServe(addr, nil)) //#nosec: g114
}
func ScheduleAndRunJobs(jobs []Job) error {
@ -110,6 +115,8 @@ func ScheduleAndRunJobs(jobs []Job) error {
defer func() {
ctx := scheduler.Stop()
<-ctx.Done()
fmt.Println("All jobs successfully stopped")
}()
return nil

View File

@ -5,6 +5,7 @@ import (
"log"
"os"
"os/exec"
"sort"
"strings"
)
@ -39,6 +40,7 @@ func NewCapturedLogWriter(logger *log.Logger) *CapturedLogWriter {
return &CapturedLogWriter{Lines: []string{}, logger: logger}
}
// Write writes the provided byte slice to the logger and stores each captured line.
func (w *CapturedLogWriter) Write(content []byte) (n int, err error) {
message := string(content)
for _, line := range strings.Split(message, "\n") {
@ -49,6 +51,33 @@ func (w *CapturedLogWriter) Write(content []byte) (n int, err error) {
return len(content), nil
}
// LinesMergedWith returns a slice of lines from this logger merged with another.
func (w CapturedLogWriter) LinesMergedWith(other CapturedLogWriter) []string {
allLines := []string{}
allLines = append(allLines, w.Lines...)
allLines = append(allLines, other.Lines...)
sort.Strings(allLines)
return allLines
}
type CapturedCommandLogWriter struct {
Stdout *CapturedLogWriter
Stderr *CapturedLogWriter
}
func NewCapturedCommandLogWriter(logger *log.Logger) *CapturedCommandLogWriter {
return &CapturedCommandLogWriter{
Stdout: NewCapturedLogWriter(logger),
Stderr: NewCapturedLogWriter(logger),
}
}
func (cclw CapturedCommandLogWriter) AllLines() []string {
return cclw.Stdout.LinesMergedWith(*cclw.Stderr)
}
func RunShell(script string, cwd string, env map[string]string, logger *log.Logger) error {
cmd := exec.Command("sh", "-c", strings.TrimSpace(script)) //nolint:gosec

192
tasks.go
View File

@ -67,7 +67,7 @@ func (t *JobTaskScript) SetName(name string) {
t.name = name
}
// JobTaskMySQL is a sqlite backup task that performs required pre and post tasks.
// JobTaskMySQL is a MySQL backup task that performs required pre and post tasks.
type JobTaskMySQL struct {
Port int `hcl:"port,optional"`
Name string `hcl:"name,label"`
@ -89,11 +89,16 @@ func (t JobTaskMySQL) Validate() error {
return fmt.Errorf("task %s is missing dump_to path: %w", t.Name, ErrMissingField)
}
if s, err := os.Stat(t.DumpToPath); err != nil {
if stat, err := os.Stat(t.DumpToPath); err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("task %s: invalid dump_to: could not stat path: %v: %w", t.Name, err, ErrInvalidConfigValue)
return fmt.Errorf(
"task %s: invalid dump_to: could not stat path: %s: %w",
t.Name,
t.DumpToPath,
ErrInvalidConfigValue,
)
}
} else if s.Mode().IsDir() {
} else if stat.Mode().IsDir() {
return fmt.Errorf("task %s: dump_to cannot be a directory: %w", t.Name, ErrInvalidConfigValue)
}
@ -155,12 +160,20 @@ func (t JobTaskMySQL) GetPostTask() ExecutableTask {
command = append(command, "--host", t.Hostname)
}
if t.Port != 0 {
command = append(command, "--port", fmt.Sprintf("%d", t.Port))
}
if t.Username != "" {
command = append(command, "--user", t.Username)
}
if t.Password != "" {
command = append(command, "--password", t.Password)
command = append(command, fmt.Sprintf("--password=%s", t.Password))
}
if t.Database != "" {
command = append(command, t.Database)
}
command = append(command, "<", t.DumpToPath)
@ -174,6 +187,144 @@ func (t JobTaskMySQL) GetPostTask() ExecutableTask {
}
}
// JobTaskPostgres is a postgres backup task that performs required pre and post tasks.
type JobTaskPostgres struct {
Port int `hcl:"port,optional"`
Name string `hcl:"name,label"`
Hostname string `hcl:"hostname,optional"`
Database string `hcl:"database,optional"`
Username string `hcl:"username,optional"`
Password string `hcl:"password,optional"`
Tables []string `hcl:"tables,optional"`
DumpToPath string `hcl:"dump_to"`
NoTablespaces bool `hcl:"no_tablespaces,optional"`
Clean bool `hcl:"clean,optional"`
Create bool `hcl:"create,optional"`
}
func (t JobTaskPostgres) Paths() []string {
return []string{t.DumpToPath}
}
func (t JobTaskPostgres) Validate() error {
if t.DumpToPath == "" {
return fmt.Errorf("task %s is missing dump_to path: %w", t.Name, ErrMissingField)
}
if stat, err := os.Stat(t.DumpToPath); err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf(
"task %s: invalid dump_to: could not stat path: %s: %w",
t.Name,
t.DumpToPath,
ErrInvalidConfigValue,
)
}
} else if stat.Mode().IsDir() {
return fmt.Errorf("task %s: dump_to cannot be a directory: %w", t.Name, ErrInvalidConfigValue)
}
if len(t.Tables) > 0 && t.Database == "" {
return fmt.Errorf(
"task %s is invalid. Must specify a database to use tables: %w",
t.Name,
ErrMissingField,
)
}
return nil
}
//nolint:cyclop
func (t JobTaskPostgres) GetPreTask() ExecutableTask {
command := []string{"pg_dump"}
if t.Database == "" {
command = []string{"pg_dumpall"}
}
command = append(command, "--file", t.DumpToPath)
if t.Hostname != "" {
command = append(command, "--host", t.Hostname)
}
if t.Port != 0 {
command = append(command, "--port", fmt.Sprintf("%d", t.Port))
}
if t.Username != "" {
command = append(command, "--username", t.Username)
}
if t.NoTablespaces {
command = append(command, "--no-tablespaces")
}
if t.Clean {
command = append(command, "--clean")
}
if t.Create {
command = append(command, "--create")
}
for _, table := range t.Tables {
command = append(command, "--table", table)
}
if t.Database != "" {
command = append(command, t.Database)
}
env := map[string]string{}
if t.Password != "" {
env["PGPASSWORD"] = t.Password
}
return JobTaskScript{
name: t.Name,
env: env,
Cwd: ".",
OnBackup: strings.Join(command, " "),
OnRestore: "",
}
}
func (t JobTaskPostgres) GetPostTask() ExecutableTask {
command := []string{"psql"}
if t.Hostname != "" {
command = append(command, "--host", t.Hostname)
}
if t.Port != 0 {
command = append(command, "--port", fmt.Sprintf("%d", t.Port))
}
if t.Username != "" {
command = append(command, "--username", t.Username)
}
if t.Database != "" {
command = append(command, t.Database)
}
command = append(command, "<", t.DumpToPath)
env := map[string]string{}
if t.Password != "" {
env["PGPASSWORD"] = t.Password
}
return JobTaskScript{
name: t.Name,
env: env,
Cwd: ".",
OnBackup: "",
OnRestore: strings.Join(command, " "),
}
}
// JobTaskSqlite is a sqlite backup task that performs required pre and post tasks.
type JobTaskSqlite struct {
Name string `hcl:"name,label"`
@ -190,11 +341,16 @@ func (t JobTaskSqlite) Validate() error {
return fmt.Errorf("task %s is missing dump_to path: %w", t.Name, ErrMissingField)
}
if s, err := os.Stat(t.DumpToPath); err != nil {
if stat, err := os.Stat(t.DumpToPath); err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("task %s: invalid dump_to: could not stat path: %v: %w", t.Name, err, ErrInvalidConfigValue)
return fmt.Errorf(
"task %s: invalid dump_to: could not stat path: %s: %w",
t.Name,
t.DumpToPath,
ErrInvalidConfigValue,
)
}
} else if s.Mode().IsDir() {
} else if stat.Mode().IsDir() {
return fmt.Errorf("task %s: dump_to cannot be a directory: %w", t.Name, ErrInvalidConfigValue)
}
@ -226,6 +382,7 @@ type BackupFilesTask struct {
BackupOpts *BackupOpts `hcl:"backup_opts,block"`
RestoreOpts *RestoreOpts `hcl:"restore_opts,block"`
name string
snapshot string
}
func (t BackupFilesTask) RunBackup(cfg TaskConfig) error {
@ -248,8 +405,11 @@ func (t BackupFilesTask) RunRestore(cfg TaskConfig) error {
t.RestoreOpts = &RestoreOpts{} //nolint:exhaustruct
}
// TODO: Make the snapshot configurable
if err := cfg.Restic.Restore("latest", *t.RestoreOpts); err != nil {
if t.snapshot == "" {
t.snapshot = "latest"
}
if err := cfg.Restic.Restore(t.snapshot, *t.RestoreOpts); err != nil {
err = fmt.Errorf("failed restoring paths: %w", err)
cfg.Logger.Print(err)
@ -277,14 +437,16 @@ func (t *BackupFilesTask) Validate() error {
// JobTask represents a single task within a backup job.
type JobTask struct {
Name string `hcl:"name,label"`
PreScripts []JobTaskScript `hcl:"pre_script,block"`
PostScripts []JobTaskScript `hcl:"post_script,block"`
MySQL []JobTaskMySQL `hcl:"mysql,block"`
Sqlite []JobTaskSqlite `hcl:"sqlite,block"`
Name string `hcl:"name,label"`
PreScripts []JobTaskScript `hcl:"pre_script,block"`
PostScripts []JobTaskScript `hcl:"post_script,block"`
MySQL []JobTaskMySQL `hcl:"mysql,block"`
Postgres []JobTaskPostgres `hcl:"postgres,block"`
Sqlite []JobTaskSqlite `hcl:"sqlite,block"`
}
func (t JobTask) Validate() error {
// NOTE: Might make task types mutually exclusive because order is confusing even if deterministic
if t.Name == "" {
return fmt.Errorf("task is missing a name: %w", ErrMissingField)
}

View File

@ -163,7 +163,29 @@ func TestJobTaskSql(t *testing.T) {
" --user user --password=pass --no-tablespaces db table1 table2",
postBackup: "",
preRestore: "",
postRestore: "mysql --host host --user user --password pass < ./simple.sql",
postRestore: "mysql --host host --port 3306 --user user --password=pass db < ./simple.sql",
},
{
name: "psql all",
task: main.JobTaskPostgres{
Name: "simple",
Hostname: "host",
Port: 6543,
Username: "user",
Password: "pass",
Database: "db",
NoTablespaces: true,
Create: true,
Clean: true,
Tables: []string{"table1", "table2"},
DumpToPath: "./simple.sql",
},
validationErr: nil,
preBackup: "pg_dump --file ./simple.sql --host host --port 6543 --username user --no-tablespaces" +
" --clean --create --table table1 --table table2 db",
postBackup: "",
preRestore: "",
postRestore: "psql --host host --port 6543 --username user db < ./simple.sql",
},
// Sqlite
{