Refactor for restore and add integration testing

This commit is contained in:
IamTheFij 2022-03-25 22:44:04 -07:00
parent a746859386
commit 97765853b4
7 changed files with 721 additions and 573 deletions

365
job.go
View File

@ -6,12 +6,11 @@ import (
"log"
"os"
"path/filepath"
"strings"
"github.com/robfig/cron/v3"
)
const WorkDirPerms = 0o666
const WorkDirPerms = 0770
var (
ErrNoJobsFound = errors.New("no jobs found and at least one job is required")
@ -19,14 +18,11 @@ var (
ErrMissingBlock = errors.New("missing config block")
ErrMutuallyExclusive = errors.New("mutually exclusive values not valid")
ErrInvalidConfigValue = errors.New("invalid config value")
)
type TaskConfig struct {
JobDir string
Env map[string]string
Logger *log.Logger
Restic *Restic
}
// JobBaseDir is the root for the creation of restic job dirs. These will generally
// house SQL dumps prior to backup and before restoration.
JobBaseDir = filepath.Join(os.TempDir(), "restic_scheduler")
)
// ResticConfig is all configuration to be sent to Restic.
type ResticConfig struct {
@ -54,290 +50,6 @@ func (r ResticConfig) Validate() error {
return nil
}
// ExecutableTask is a task to be run before or after backup/retore.
type ExecutableTask interface {
RunBackup(cfg TaskConfig) error
RunRestore(cfg TaskConfig) error
Name() string
}
// JobTaskScript is a sript to be executed as part of a job task.
type JobTaskScript struct {
OnBackup string `hcl:"on_backup,optional"`
OnRestore string `hcl:"on_restore,optional"`
FromJobDir bool `hcl:"from_job_dir,optional"`
env map[string]string
name string
}
func (t JobTaskScript) run(script string, cfg TaskConfig) error {
if script == "" {
return nil
}
env := MergeEnvMap(cfg.Env, t.env)
if env == nil {
env = map[string]string{}
}
// Inject the job directory to the running task
env["RESTIC_JOB_DIR"] = cfg.JobDir
cwd := ""
if t.FromJobDir {
cwd = cfg.JobDir
}
if err := RunShell(script, cwd, env, cfg.Logger); err != nil {
return fmt.Errorf("failed running task script %s: %w", t.Name(), err)
}
return nil
}
// RunBackup runs script on backup.
func (t JobTaskScript) RunBackup(cfg TaskConfig) error {
return t.run(t.OnBackup, cfg)
}
// RunRestore script on restore.
func (t JobTaskScript) RunRestore(cfg TaskConfig) error {
return t.run(t.OnRestore, cfg)
}
func (t JobTaskScript) Name() string {
return t.name
}
func (t *JobTaskScript) SetName(name string) {
t.name = name
}
// JobTaskMySQL is a sqlite backup task that performs required pre and post tasks.
type JobTaskMySQL struct {
Name string `hcl:"name,label"`
Hostname string `hcl:"hostname,optional"`
Database string `hcl:"database,optional"`
Username string `hcl:"username,optional"`
Password string `hcl:"password,optional"`
Tables []string `hcl:"tables,optional"`
}
func (t JobTaskMySQL) Filename() string {
return fmt.Sprintf("%s.sql", t.Name)
}
func (t JobTaskMySQL) Validate() error {
if invalidChars := "'\";"; strings.ContainsAny(t.Name, invalidChars) {
return fmt.Errorf(
"mysql task %s has an invalid name. The name may not contain %s: %w",
t.Name,
invalidChars,
ErrInvalidConfigValue,
)
}
if len(t.Tables) > 0 && t.Database == "" {
return fmt.Errorf(
"mysql task %s is invalid. Must specify a database to use tables: %w",
t.Name,
ErrMissingField,
)
}
return nil
}
func (t JobTaskMySQL) GetPreTask() ExecutableTask {
command := []string{"mysqldump", "--result-file", fmt.Sprintf("'./%s'", t.Filename())}
if t.Hostname != "" {
command = append(command, "--host", t.Hostname)
}
if t.Username != "" {
command = append(command, "--user", t.Username)
}
if t.Password != "" {
command = append(command, "--password", t.Password)
}
if t.Database != "" {
command = append(command, t.Database)
}
command = append(command, t.Tables...)
return JobTaskScript{
name: t.Name,
env: nil,
OnBackup: strings.Join(command, " "),
OnRestore: "",
FromJobDir: true,
}
}
func (t JobTaskMySQL) GetPostTask() ExecutableTask {
command := []string{"mysql"}
if t.Hostname != "" {
command = append(command, "--host", t.Hostname)
}
if t.Username != "" {
command = append(command, "--user", t.Username)
}
if t.Password != "" {
command = append(command, "--password", t.Password)
}
command = append(command, "<", fmt.Sprintf("'./%s'", t.Filename()))
return JobTaskScript{
name: t.Name,
env: nil,
OnBackup: "",
OnRestore: strings.Join(command, " "),
FromJobDir: true,
}
}
// JobTaskSqlite is a sqlite backup task that performs required pre and post tasks.
type JobTaskSqlite struct {
Name string `hcl:"name,label"`
Path string `hcl:"path"`
}
func (t JobTaskSqlite) Filename() string {
return fmt.Sprintf("%s.db.bak", t.Name)
}
func (t JobTaskSqlite) Validate() error {
if invalidChars := "'\";"; strings.ContainsAny(t.Name, invalidChars) {
return fmt.Errorf(
"sqlite task %s has an invalid name. The name may not contain %s: %w",
t.Name,
invalidChars,
ErrInvalidConfigValue,
)
}
return nil
}
func (t JobTaskSqlite) GetPreTask() ExecutableTask {
return JobTaskScript{
name: t.Name,
env: nil,
OnBackup: fmt.Sprintf(
"sqlite3 '%s' '.backup $RESTIC_JOB_DIR/%s'",
t.Path, t.Filename(),
),
OnRestore: "",
FromJobDir: false,
}
}
func (t JobTaskSqlite) GetPostTask() ExecutableTask {
return JobTaskScript{
name: t.Name,
env: nil,
OnBackup: "",
OnRestore: fmt.Sprintf("cp '$RESTIC_JOB_DIR/%s' '%s'", t.Filename(), t.Path),
FromJobDir: false,
}
}
type BackupFilesTask struct {
Files []string `hcl:"files"`
BackupOpts *BackupOpts `hcl:"backup_opts,block"`
RestoreOpts *RestoreOpts `hcl:"restore_opts,block"`
name string
}
func (t BackupFilesTask) RunBackup(cfg TaskConfig) error {
if t.BackupOpts == nil {
t.BackupOpts = &BackupOpts{} // nolint:exhaustivestruct
}
if err := cfg.Restic.Backup(append(t.Files, cfg.JobDir), *t.BackupOpts); err != nil {
err = fmt.Errorf("failed backing up files: %w", err)
cfg.Logger.Fatal(err)
return err
}
return nil
}
func (t BackupFilesTask) RunRestore(cfg TaskConfig) error {
if t.RestoreOpts == nil {
t.RestoreOpts = &RestoreOpts{} // nolint:exhaustivestruct
}
if err := cfg.Restic.Restore("latest", *t.RestoreOpts); err != nil {
err = fmt.Errorf("failed restoring files: %w", err)
cfg.Logger.Fatal(err)
return err
}
return nil
}
func (t BackupFilesTask) Name() string {
return t.name
}
func (t *BackupFilesTask) SetName(name string) {
t.name = name
}
// JobTask represents a single task within a backup job.
type JobTask struct {
Name string `hcl:"name,label"`
Scripts []JobTaskScript `hcl:"script,block"`
Backup *BackupFilesTask `hcl:"backup,block"`
}
func (t JobTask) Validate() error {
if len(t.Scripts) > 0 && t.Backup != nil {
return fmt.Errorf(
"task %s is invalid. script and backup blocks are mutually exclusive: %w",
t.Name,
ErrMutuallyExclusive,
)
}
if len(t.Scripts) == 0 && t.Backup == nil {
return fmt.Errorf(
"task %s is invalid. Ether script or backup blocks must be provided: %w",
t.Name,
ErrMutuallyExclusive,
)
}
return nil
}
func (t JobTask) GetTasks() []ExecutableTask {
allTasks := []ExecutableTask{}
for _, exTask := range t.Scripts {
exTask.SetName(t.Name)
allTasks = append(allTasks, exTask)
}
if t.Backup != nil {
t.Backup.SetName(t.Name)
allTasks = append(allTasks, t.Backup)
}
return allTasks
}
// Job contains all configuration required to construct and run a backup
// and restore job.
type Job struct {
@ -345,11 +57,15 @@ type Job struct {
Schedule string `hcl:"schedule"`
Config ResticConfig `hcl:"config,block"`
Tasks []JobTask `hcl:"task,block"`
Backup BackupFilesTask `hcl:"backup,block"`
Forget *ForgetOpts `hcl:"forget,block"`
// Meta Tasks
MySQL []JobTaskMySQL `hcl:"mysql,block"`
Sqlite []JobTaskSqlite `hcl:"sqlite,block"`
healthy bool
lastErr error
}
func (j Job) validateTasks() error {
@ -357,22 +73,12 @@ func (j Job) validateTasks() error {
return fmt.Errorf("job %s is missing tasks: %w", j.Name, ErrMissingBlock)
}
foundBackup := false
for _, task := range j.Tasks {
if task.Backup != nil {
foundBackup = true
}
if err := task.Validate(); err != nil {
return fmt.Errorf("job %s has an inavalid task: %w", j.Name, err)
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
if !foundBackup {
return fmt.Errorf("job %s is missing a backup task: %w", j.Name, ErrMissingBlock)
}
return nil
}
@ -395,13 +101,13 @@ func (j Job) Validate() error {
for _, mysql := range j.MySQL {
if err := mysql.Validate(); err != nil {
return fmt.Errorf("job %s has an inavalid task: %w", j.Name, err)
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
for _, sqlite := range j.Sqlite {
if err := sqlite.Validate(); err != nil {
return fmt.Errorf("job %s has an inavalid task: %w", j.Name, err)
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
@ -420,42 +126,63 @@ func (j Job) AllTasks() []ExecutableTask {
allTasks = append(allTasks, sqlite.GetPreTask())
}
// Get ordered tasks
for _, jobTask := range j.Tasks {
allTasks = append(allTasks, jobTask.GetTasks()...)
allTasks = append(allTasks, jobTask.GetPreTasks()...)
}
// Add backup task
allTasks = append(allTasks, j.Backup)
// Post tasks
for _, jobTask := range j.Tasks {
allTasks = append(allTasks, jobTask.GetPostTasks()...)
}
for _, mysql := range j.MySQL {
allTasks = append(allTasks, mysql.GetPreTask())
allTasks = append(allTasks, mysql.GetPostTask())
}
for _, sqlite := range j.Sqlite {
allTasks = append(allTasks, sqlite.GetPreTask())
allTasks = append(allTasks, sqlite.GetPostTask())
}
return allTasks
}
func (j Job) JobDir() string {
cwd := filepath.Join("/restic_backup", j.Name)
cwd := filepath.Join(JobBaseDir, j.Name)
_ = os.MkdirAll(cwd, WorkDirPerms)
return cwd
}
func (j Job) BackupPaths() []string {
files := j.Backup.Files
for _, t := range j.MySQL {
files = append(files, t.DumpToPath)
}
for _, t := range j.Sqlite {
files = append(files, t.DumpToPath)
}
return files
}
func (j Job) RunBackup() error {
logger := GetLogger(j.Name)
restic := j.NewRestic()
jobDir := j.JobDir()
if err := restic.EnsureInit(); err != nil {
return fmt.Errorf("failed to init restic for job %s: %w", j.Name, err)
}
backupPaths := j.BackupPaths()
for _, exTask := range j.AllTasks() {
taskCfg := TaskConfig{
JobDir: jobDir,
BackupPaths: backupPaths,
Logger: GetChildLogger(logger, exTask.Name()),
Restic: restic,
Env: nil,
@ -482,7 +209,6 @@ func (j Job) Logger() *log.Logger {
func (j Job) RunRestore() error {
logger := j.Logger()
restic := j.NewRestic()
jobDir := j.JobDir()
if _, err := restic.RunRestic("snapshots", NoOpts{}); errors.Is(err, ErrRepoNotFound) {
return fmt.Errorf("no repository or snapshots for job %s: %w", j.Name, err)
@ -490,7 +216,7 @@ func (j Job) RunRestore() error {
for _, exTask := range j.AllTasks() {
taskCfg := TaskConfig{
JobDir: jobDir,
BackupPaths: nil,
Logger: GetChildLogger(logger, exTask.Name()),
Restic: restic,
Env: nil,
@ -504,9 +230,16 @@ func (j Job) RunRestore() error {
return nil
}
func (j Job) Healthy() (bool, error) {
return j.healthy, j.lastErr
}
func (j Job) Run() {
if err := j.RunBackup(); err != nil {
j.Logger().Fatalf("ERROR: Backup failed: %v", err)
j.healthy = false
j.lastErr = err
j.Logger().Printf("ERROR: Backup failed: %v", err)
}
}

View File

@ -1,9 +1,7 @@
package main_test
import (
"bytes"
"errors"
"log"
"testing"
main "git.iamthefij.com/iamthefij/restic-scheduler"
@ -66,232 +64,3 @@ func TestResticConfigValidate(t *testing.T) {
})
}
}
func NewBufferedLogger(prefix string) (*bytes.Buffer, *log.Logger) {
outputBuffer := bytes.Buffer{}
logger := log.New(&outputBuffer, prefix, 0)
return &outputBuffer, logger
}
func TestJobTaskScript(t *testing.T) {
t.Parallel()
cases := []struct {
name string
script main.JobTaskScript
config main.TaskConfig
expectedErr error
expectedOutput string
}{
{
name: "simple",
config: main.TaskConfig{
JobDir: "./test",
Env: nil,
Logger: nil,
Restic: nil,
},
script: main.JobTaskScript{
OnBackup: "echo yass",
OnRestore: "echo yass",
FromJobDir: false,
},
expectedErr: nil,
expectedOutput: "t yass\nt \n",
},
{
name: "check job dir",
config: main.TaskConfig{
JobDir: "./test",
Env: nil,
Logger: nil,
Restic: nil,
},
script: main.JobTaskScript{
OnBackup: "echo $RESTIC_JOB_DIR",
OnRestore: "echo $RESTIC_JOB_DIR",
FromJobDir: false,
},
expectedErr: nil,
expectedOutput: "t ./test\nt \n",
},
{
name: "check from job dir",
config: main.TaskConfig{
JobDir: "./test",
Env: nil,
Logger: nil,
Restic: nil,
},
script: main.JobTaskScript{
OnBackup: "basename `pwd`",
OnRestore: "basename `pwd`",
FromJobDir: true,
},
expectedErr: nil,
expectedOutput: "t test\nt \n",
},
{
name: "check env",
config: main.TaskConfig{
JobDir: "./test",
Env: map[string]string{"TEST": "OK"},
Logger: nil,
Restic: nil,
},
script: main.JobTaskScript{
OnBackup: "echo $TEST",
OnRestore: "echo $TEST",
FromJobDir: false,
},
expectedErr: nil,
expectedOutput: "t OK\nt \n",
},
}
for _, c := range cases {
testCase := c
buf, logger := NewBufferedLogger("t")
testCase.config.Logger = logger
t.Run(testCase.name, func(t *testing.T) {
t.Parallel()
actual := testCase.script.RunBackup(testCase.config)
if !errors.Is(actual, testCase.expectedErr) {
t.Errorf("expected error to wrap %v but found %v", testCase.expectedErr, actual)
}
output := buf.String()
if testCase.expectedOutput != output {
t.Errorf("Unexpected output. expected: %s actual: %s", testCase.expectedOutput, output)
}
})
}
}
func TestJobTaskSql(t *testing.T) {
t.Parallel()
type TaskGenerator interface {
Validate() error
GetPreTask() main.ExecutableTask
GetPostTask() main.ExecutableTask
}
cases := []struct {
name string
task TaskGenerator
validationErr error
preBackup string
postBackup string
preRestore string
postRestore string
}{
{
name: "mysql simple",
// nolint:exhaustivestruct
task: main.JobTaskMySQL{Name: "simple"},
validationErr: nil,
preBackup: "mysqldump --result-file './simple.sql'",
postBackup: "",
preRestore: "",
postRestore: "mysql < './simple.sql'",
},
{
name: "mysql invalid name",
// nolint:exhaustivestruct
task: main.JobTaskMySQL{Name: "it's invalid;"},
validationErr: main.ErrInvalidConfigValue,
preBackup: "",
postBackup: "",
preRestore: "",
postRestore: "",
},
{
name: "mysql tables no database",
// nolint:exhaustivestruct
task: main.JobTaskMySQL{
Name: "name",
Tables: []string{"table1", "table2"},
},
validationErr: main.ErrMissingField,
preBackup: "",
postBackup: "",
preRestore: "",
postRestore: "",
},
{
name: "mysql all options",
task: main.JobTaskMySQL{
Name: "simple",
Hostname: "host",
Username: "user",
Password: "pass",
Database: "db",
Tables: []string{"table1", "table2"},
},
validationErr: nil,
preBackup: "mysqldump --result-file './simple.sql' --host host --user user --password pass db table1 table2",
postBackup: "",
preRestore: "",
postRestore: "mysql --host host --user user --password pass < './simple.sql'",
},
// Sqlite
{
name: "sqlite simple",
task: main.JobTaskSqlite{Name: "simple", Path: "database.db"},
validationErr: nil,
preBackup: "sqlite3 'database.db' '.backup $RESTIC_JOB_DIR/simple.db.bak'",
postBackup: "",
preRestore: "",
postRestore: "cp '$RESTIC_JOB_DIR/simple.db.bak' 'database.db'",
},
{
name: "sqlite invalid name",
task: main.JobTaskSqlite{Name: "it's invalid;", Path: "database.db"},
validationErr: main.ErrInvalidConfigValue,
preBackup: "",
postBackup: "",
preRestore: "",
postRestore: "",
},
}
for _, c := range cases {
testCase := c
t.Run(testCase.name, func(t *testing.T) {
t.Parallel()
validateErr := testCase.task.Validate()
if !errors.Is(validateErr, testCase.validationErr) {
t.Errorf("unexpected validation result. expected: %v, actual: %v", testCase.validationErr, validateErr)
}
if validateErr != nil {
return
}
if preTask, ok := testCase.task.GetPreTask().(main.JobTaskScript); ok {
AssertEqual(t, "incorrect pre-backup", testCase.preBackup, preTask.OnBackup)
AssertEqual(t, "incorrect pre-restore", testCase.preRestore, preTask.OnRestore)
} else {
t.Error("pre task was not a JobTaskScript")
}
if postTask, ok := testCase.task.GetPostTask().(main.JobTaskScript); ok {
AssertEqual(t, "incorrect post-backup", testCase.postBackup, postTask.OnBackup)
AssertEqual(t, "incorrect post-restore", testCase.postRestore, postTask.OnRestore)
} else {
t.Error("post task was not a JobTaskScript")
}
})
}
}

125
main.go
View File

@ -4,6 +4,7 @@ import (
"flag"
"fmt"
"log"
"strings"
"github.com/hashicorp/hcl/v2/hclsimple"
)
@ -13,8 +14,92 @@ var (
version = "dev"
)
func parseConfig(path string) ([]Job, error) {
var config Config
if err := hclsimple.DecodeFile(path, nil, &config); err != nil {
return nil, fmt.Errorf("%s: Failed to decode file: %w", path, err)
}
if len(config.Jobs) == 0 {
log.Printf("%s: No jobs defined in file", path)
return []Job{}, nil
}
for _, job := range config.Jobs {
if err := job.Validate(); err != nil {
return nil, fmt.Errorf("%s: Invalid job: %w", path, err)
}
}
return config.Jobs, nil
}
func readJobs(paths []string) ([]Job, error) {
allJobs := []Job{}
for _, path := range paths {
jobs, err := parseConfig(path)
if err != nil {
return nil, err
}
if jobs != nil {
allJobs = append(allJobs, jobs...)
}
}
return allJobs, nil
}
type Set map[string]bool
func NewSetFrom(l []string) Set {
s := make(Set)
for _, l := range l {
s[l] = true
}
return s
}
func runBackupJobs(jobs []Job, names []string) error {
nameSet := NewSetFrom(names)
_, runAll := nameSet["all"]
for _, job := range jobs {
if _, found := nameSet[job.Name]; runAll || found {
if err := job.RunBackup(); err != nil {
return err
}
}
}
return nil
}
func runRestoreJobs(jobs []Job, names []string) error {
nameSet := NewSetFrom(names)
_, runAll := nameSet["all"]
for _, job := range jobs {
if _, found := nameSet[job.Name]; runAll || found {
if err := job.RunRestore(); err != nil {
return err
}
}
}
return nil
}
func main() {
showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
showVersion := flag.Bool("version", false, "Display the version and exit")
backup := flag.String("backup", "", "Run backup jobs now. Names are comma separated and `all` will run all.")
restore := flag.String("restore", "", "Run restore jobs now. Names are comma separated and `all` will run all.")
once := flag.Bool("once", false, "Run jobs specified using -backup and -restore once and exit")
flag.StringVar(&JobBaseDir, "base-dir", JobBaseDir, "Base dir to create intermediate job files like SQL dumps.")
flag.Parse()
// Print version if flag is provided
@ -24,26 +109,36 @@ func main() {
return
}
var config Config
args := flag.Args()
if len(args) == 0 {
if flag.NArg() == 0 {
log.Fatalf("Requires a path to a job file, but found none")
}
if err := hclsimple.DecodeFile(args[0], nil, &config); err != nil {
log.Fatalf("Failed to load configuration: %s", err)
jobs, err := readJobs(flag.Args())
if err != nil {
log.Fatalf("Failed to read jobs from files: %v", err)
}
log.Printf("Configuration is %#v", config)
if len(config.Jobs) == 0 {
log.Fatalf("No jobs defined in config")
if len(jobs) == 0 {
log.Fatal("No jobs found in provided configuration")
}
for _, job := range config.Jobs {
if err := job.RunBackup(); err != nil {
log.Fatalf("%v", err)
}
// Run specified backup jobs
if err := runBackupJobs(jobs, strings.Split(*backup, ",")); err != nil {
log.Fatalf("Failed running backup jobs: %v", err)
}
// Run specified restore jobs
if err := runRestoreJobs(jobs, strings.Split(*restore, ",")); err != nil {
log.Fatalf("Failed running backup jobs: %v", err)
}
// Exit if only running once
if *once {
return
}
// TODO: Add healthcheck handler using Job.Healthy()
if err := ScheduleAndRunJobs(jobs); err != nil {
log.Fatalf("failed running jobs: %v", err)
}
}

55
scheduler.go Normal file
View File

@ -0,0 +1,55 @@
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
"github.com/robfig/cron/v3"
)
func ScheduleAndRunJobs(jobs []Job) error {
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT,
)
runner := cron.New()
for _, job := range jobs {
fmt.Println("Scheduling", job.Name)
if _, err := runner.AddJob(job.Schedule, job); err != nil {
return fmt.Errorf("Error scheduling job %s: %w", job.Name, err)
}
}
runner.Start()
switch <-signalChan {
case syscall.SIGINT:
fmt.Println("Stopping now...")
defer runner.Stop()
return nil
case syscall.SIGTERM:
fallthrough
case syscall.SIGQUIT:
// Wait for all jobs to complete
fmt.Println("Stopping after running jobs complete...")
defer func() {
ctx := runner.Stop()
<-ctx.Done()
}()
return nil
}
return nil
}

281
tasks.go Normal file
View File

@ -0,0 +1,281 @@
package main
import (
"errors"
"fmt"
"io/fs"
"log"
"os"
"strings"
)
type TaskConfig struct {
BackupPaths []string
Env map[string]string
Logger *log.Logger
Restic *Restic
}
// ExecutableTask is a task to be run before or after backup/retore.
type ExecutableTask interface {
RunBackup(cfg TaskConfig) error
RunRestore(cfg TaskConfig) error
Name() string
}
// JobTaskScript is a sript to be executed as part of a job task.
type JobTaskScript struct {
OnBackup string `hcl:"on_backup,optional"`
OnRestore string `hcl:"on_restore,optional"`
Cwd string `hcl:"cwd,optional"`
env map[string]string
name string
}
func (t JobTaskScript) run(script string, cfg TaskConfig) error {
if script == "" {
return nil
}
env := MergeEnvMap(cfg.Env, t.env)
if env == nil {
env = map[string]string{}
}
if err := RunShell(script, t.Cwd, env, cfg.Logger); err != nil {
return fmt.Errorf("failed running task script %s: %w", t.Name(), err)
}
return nil
}
// RunBackup runs script on backup.
func (t JobTaskScript) RunBackup(cfg TaskConfig) error {
return t.run(t.OnBackup, cfg)
}
// RunRestore script on restore.
func (t JobTaskScript) RunRestore(cfg TaskConfig) error {
return t.run(t.OnRestore, cfg)
}
func (t JobTaskScript) Name() string {
return t.name
}
func (t *JobTaskScript) SetName(name string) {
t.name = name
}
// JobTaskMySQL is a sqlite backup task that performs required pre and post tasks.
type JobTaskMySQL struct {
Name string `hcl:"name,label"`
Hostname string `hcl:"hostname,optional"`
Database string `hcl:"database,optional"`
Username string `hcl:"username,optional"`
Password string `hcl:"password,optional"`
Tables []string `hcl:"tables,optional"`
DumpToPath string `hcl:"dump_to"`
}
func (t JobTaskMySQL) Paths() []string {
return []string{t.DumpToPath}
}
func (t JobTaskMySQL) Validate() error {
if s, err := os.Stat(t.DumpToPath); err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("Could not stat dump file path: %w", err)
}
} else if s.Mode().IsDir() {
return fmt.Errorf("dump_to cannot be a directory: %w", ErrInvalidConfigValue)
}
if len(t.Tables) > 0 && t.Database == "" {
return fmt.Errorf(
"mysql task %s is invalid. Must specify a database to use tables: %w",
t.Name,
ErrMissingField,
)
}
return nil
}
func (t JobTaskMySQL) GetPreTask() ExecutableTask {
command := []string{"mysqldump", "--result-file", t.DumpToPath}
if t.Hostname != "" {
command = append(command, "--host", t.Hostname)
}
if t.Username != "" {
command = append(command, "--user", t.Username)
}
if t.Password != "" {
command = append(command, "--password", t.Password)
}
if t.Database != "" {
command = append(command, t.Database)
}
command = append(command, t.Tables...)
return JobTaskScript{
name: t.Name,
env: nil,
Cwd: ".",
OnBackup: strings.Join(command, " "),
OnRestore: "",
}
}
func (t JobTaskMySQL) GetPostTask() ExecutableTask {
command := []string{"mysql"}
if t.Hostname != "" {
command = append(command, "--host", t.Hostname)
}
if t.Username != "" {
command = append(command, "--user", t.Username)
}
if t.Password != "" {
command = append(command, "--password", t.Password)
}
command = append(command, "<", t.DumpToPath)
return JobTaskScript{
name: t.Name,
env: nil,
Cwd: ".",
OnBackup: "",
OnRestore: strings.Join(command, " "),
}
}
// JobTaskSqlite is a sqlite backup task that performs required pre and post tasks.
type JobTaskSqlite struct {
Name string `hcl:"name,label"`
Path string `hcl:"path"`
DumpToPath string `hcl:"dump_to"`
}
func (t JobTaskSqlite) Paths() []string {
return []string{t.DumpToPath}
}
func (t JobTaskSqlite) Validate() error {
if s, err := os.Stat(t.DumpToPath); err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("Could not stat dump file path: %w", err)
}
} else if s.Mode().IsDir() {
return fmt.Errorf("dump_to cannot be a directory: %w", ErrInvalidConfigValue)
}
return nil
}
func (t JobTaskSqlite) GetPreTask() ExecutableTask {
return JobTaskScript{
name: t.Name,
env: nil,
Cwd: ".",
OnBackup: fmt.Sprintf("sqlite3 '%s' '.backup %s'", t.Path, t.DumpToPath),
OnRestore: "",
}
}
func (t JobTaskSqlite) GetPostTask() ExecutableTask {
return JobTaskScript{
name: t.Name,
env: nil,
Cwd: ".",
OnBackup: "",
OnRestore: fmt.Sprintf("cp '%s' '%s'", t.DumpToPath, t.Path),
}
}
type BackupFilesTask struct {
Files []string `hcl:"files"`
BackupOpts *BackupOpts `hcl:"backup_opts,block"`
RestoreOpts *RestoreOpts `hcl:"restore_opts,block"`
name string
}
func (t BackupFilesTask) RunBackup(cfg TaskConfig) error {
if t.BackupOpts == nil {
t.BackupOpts = &BackupOpts{} // nolint:exhaustivestruct
}
if err := cfg.Restic.Backup(cfg.BackupPaths, *t.BackupOpts); err != nil {
err = fmt.Errorf("failed backing up files: %w", err)
cfg.Logger.Fatal(err)
return err
}
return nil
}
func (t BackupFilesTask) RunRestore(cfg TaskConfig) error {
if t.RestoreOpts == nil {
t.RestoreOpts = &RestoreOpts{} // nolint:exhaustivestruct
}
// TODO: Make the snapshot configurable
if err := cfg.Restic.Restore("latest", *t.RestoreOpts); err != nil {
err = fmt.Errorf("failed restoring files: %w", err)
cfg.Logger.Fatal(err)
return err
}
return nil
}
func (t BackupFilesTask) Name() string {
return t.name
}
func (t *BackupFilesTask) SetName(name string) {
t.name = name
}
// JobTask represents a single task within a backup job.
type JobTask struct {
Name string `hcl:"name,label"`
PreScripts []JobTaskScript `hcl:"pre_script,block"`
PostScripts []JobTaskScript `hcl:"post_script,block"`
}
func (t JobTask) Validate() error {
return nil
}
func (t JobTask) GetPreTasks() []ExecutableTask {
allTasks := []ExecutableTask{}
for _, exTask := range t.PreScripts {
exTask.SetName(t.Name)
allTasks = append(allTasks, exTask)
}
return allTasks
}
func (t JobTask) GetPostTasks() []ExecutableTask {
allTasks := []ExecutableTask{}
for _, exTask := range t.PostScripts {
exTask.SetName(t.Name)
allTasks = append(allTasks, exTask)
}
return allTasks
}

211
tasks_test.go Normal file
View File

@ -0,0 +1,211 @@
package main_test
import (
"bytes"
"errors"
"log"
"testing"
main "git.iamthefij.com/iamthefij/restic-scheduler"
)
func NewBufferedLogger(prefix string) (*bytes.Buffer, *log.Logger) {
outputBuffer := bytes.Buffer{}
logger := log.New(&outputBuffer, prefix, 0)
return &outputBuffer, logger
}
func TestJobTaskScript(t *testing.T) {
t.Parallel()
cases := []struct {
name string
script main.JobTaskScript
config main.TaskConfig
expectedErr error
expectedOutput string
}{
{
name: "simple",
config: main.TaskConfig{
BackupPaths: nil,
Env: nil,
Logger: nil,
Restic: nil,
},
script: main.JobTaskScript{
Cwd: "./test",
OnBackup: "echo yass",
OnRestore: "echo yass",
},
expectedErr: nil,
expectedOutput: "t yass\nt \n",
},
{
name: "check from job dir",
config: main.TaskConfig{
BackupPaths: nil,
Env: nil,
Logger: nil,
Restic: nil,
},
script: main.JobTaskScript{
Cwd: "./test",
OnBackup: "basename `pwd`",
OnRestore: "basename `pwd`",
},
expectedErr: nil,
expectedOutput: "t test\nt \n",
},
{
name: "check env",
config: main.TaskConfig{
BackupPaths: nil,
Env: map[string]string{"TEST": "OK"},
Logger: nil,
Restic: nil,
},
script: main.JobTaskScript{
Cwd: "./test",
OnBackup: "echo $TEST",
OnRestore: "echo $TEST",
},
expectedErr: nil,
expectedOutput: "t OK\nt \n",
},
}
for _, c := range cases {
testCase := c
buf, logger := NewBufferedLogger("t")
testCase.config.Logger = logger
t.Run(testCase.name, func(t *testing.T) {
t.Parallel()
actual := testCase.script.RunBackup(testCase.config)
if !errors.Is(actual, testCase.expectedErr) {
t.Errorf("expected error to wrap %v but found %v", testCase.expectedErr, actual)
}
output := buf.String()
if testCase.expectedOutput != output {
t.Errorf("Unexpected output. expected: %s actual: %s", testCase.expectedOutput, output)
}
})
}
}
func TestJobTaskSql(t *testing.T) {
t.Parallel()
type TaskGenerator interface {
Validate() error
GetPreTask() main.ExecutableTask
GetPostTask() main.ExecutableTask
}
cases := []struct {
name string
task TaskGenerator
validationErr error
preBackup string
postBackup string
preRestore string
postRestore string
}{
{
name: "mysql simple",
// nolint:exhaustivestruct
task: main.JobTaskMySQL{
Name: "simple",
DumpToPath: "./simple.sql",
},
validationErr: nil,
preBackup: "mysqldump --result-file ./simple.sql",
postBackup: "",
preRestore: "",
postRestore: "mysql < ./simple.sql",
},
{
name: "mysql tables no database",
// nolint:exhaustivestruct
task: main.JobTaskMySQL{
Name: "name",
Tables: []string{"table1", "table2"},
DumpToPath: "./simple.sql",
},
validationErr: main.ErrMissingField,
preBackup: "",
postBackup: "",
preRestore: "",
postRestore: "",
},
{
name: "mysql all options",
task: main.JobTaskMySQL{
Name: "simple",
Hostname: "host",
Username: "user",
Password: "pass",
Database: "db",
Tables: []string{"table1", "table2"},
DumpToPath: "./simple.sql",
},
validationErr: nil,
preBackup: "mysqldump --result-file ./simple.sql --host host --user user --password pass db table1 table2",
postBackup: "",
preRestore: "",
postRestore: "mysql --host host --user user --password pass < ./simple.sql",
},
// Sqlite
{
name: "sqlite simple",
task: main.JobTaskSqlite{
Name: "simple",
Path: "database.db",
DumpToPath: "./simple.db.bak",
},
validationErr: nil,
preBackup: "sqlite3 'database.db' '.backup ./simple.db.bak'",
postBackup: "",
preRestore: "",
postRestore: "cp './simple.db.bak' 'database.db'",
},
}
for _, c := range cases {
testCase := c
t.Run(testCase.name, func(t *testing.T) {
t.Parallel()
validateErr := testCase.task.Validate()
if !errors.Is(validateErr, testCase.validationErr) {
t.Errorf("unexpected validation result. expected: %v, actual: %v", testCase.validationErr, validateErr)
}
if validateErr != nil {
return
}
if preTask, ok := testCase.task.GetPreTask().(main.JobTaskScript); ok {
AssertEqual(t, "incorrect pre-backup", testCase.preBackup, preTask.OnBackup)
AssertEqual(t, "incorrect pre-restore", testCase.preRestore, preTask.OnRestore)
} else {
t.Error("pre task was not a JobTaskScript")
}
if postTask, ok := testCase.task.GetPostTask().(main.JobTaskScript); ok {
AssertEqual(t, "incorrect post-backup", testCase.postBackup, postTask.OnBackup)
AssertEqual(t, "incorrect post-restore", testCase.postRestore, postTask.OnRestore)
} else {
t.Error("post task was not a JobTaskScript")
}
})
}
}

View File

@ -25,6 +25,10 @@ job "TestBackup" {
backup_opts {
Tags = ["foo"]
}
restore_opts {
Target = "."
}
}
}