restic-scheduler/job.go

333 lines
7.8 KiB
Go
Raw Permalink Normal View History

2022-02-18 22:37:51 +00:00
package main
2022-02-23 00:39:01 +00:00
import (
2022-02-23 22:13:00 +00:00
"errors"
2022-02-23 00:39:01 +00:00
"fmt"
"log"
"os"
"path/filepath"
2022-02-23 22:13:00 +00:00
"github.com/robfig/cron/v3"
2022-02-23 00:39:01 +00:00
)
2022-02-23 22:13:00 +00:00
var (
2022-02-24 05:53:48 +00:00
ErrNoJobsFound = errors.New("no jobs found and at least one job is required")
ErrMissingField = errors.New("missing config field")
ErrMutuallyExclusive = errors.New("mutually exclusive values not valid")
ErrInvalidConfigValue = errors.New("invalid config value")
2022-02-23 22:13:00 +00:00
// JobBaseDir is the root for the creation of restic job dirs. These will generally
// house SQL dumps prior to backup and before restoration.
JobBaseDir = filepath.Join(os.TempDir(), "restic_scheduler")
)
2022-02-23 00:39:01 +00:00
2022-02-23 00:55:41 +00:00
// ResticConfig is all configuration to be sent to Restic.
2022-02-23 00:39:01 +00:00
type ResticConfig struct {
2022-02-18 22:37:51 +00:00
Repo string `hcl:"repo"`
2022-02-23 00:39:01 +00:00
Passphrase string `hcl:"passphrase,optional"`
2022-02-18 22:37:51 +00:00
Env map[string]string `hcl:"env,optional"`
2022-02-23 00:39:01 +00:00
GlobalOpts *ResticGlobalOpts `hcl:"options,block"`
}
2022-02-23 22:13:00 +00:00
func (r ResticConfig) Validate() error {
if r.Passphrase == "" && (r.GlobalOpts == nil || r.GlobalOpts.PasswordFile == "") {
return fmt.Errorf(
"either config { Passphrase = string } or config { options { PasswordFile = string } } must be set: %w",
ErrMutuallyExclusive,
)
}
if r.Passphrase != "" && r.GlobalOpts != nil && r.GlobalOpts.PasswordFile != "" {
return fmt.Errorf(
"only one of config { Passphrase = string } or config { options { PasswordFile = string } } may be set: %w",
ErrMutuallyExclusive,
)
}
return nil
}
2022-02-18 22:37:51 +00:00
// Job contains all configuration required to construct and run a backup
2022-02-23 00:55:41 +00:00
// and restore job.
2022-02-18 22:37:51 +00:00
type Job struct {
Name string `hcl:"name,label"`
Schedule string `hcl:"schedule"`
Config *ResticConfig `hcl:"config,block"`
Tasks []JobTask `hcl:"task,block"`
Backup BackupFilesTask `hcl:"backup,block"`
Forget *ForgetOpts `hcl:"forget,block"`
2022-02-23 00:39:01 +00:00
// Meta Tasks
2022-11-28 05:20:13 +00:00
// NOTE: Now that these are also available within a task
// these could be removed to make task order more obvious
MySQL []JobTaskMySQL `hcl:"mysql,block"`
Postgres []JobTaskPostgres `hcl:"postgres,block"`
Sqlite []JobTaskSqlite `hcl:"sqlite,block"`
2022-04-13 20:44:48 +00:00
// Metrics and health
healthy bool
lastErr error
2022-02-23 00:39:01 +00:00
}
2022-02-23 22:13:00 +00:00
func (j Job) validateTasks() error {
for _, task := range j.Tasks {
if err := task.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
2022-02-23 22:13:00 +00:00
}
}
for _, mysql := range j.MySQL {
if err := mysql.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
for _, pg := range j.Postgres {
if err := pg.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
for _, sqlite := range j.Sqlite {
if err := sqlite.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
}
}
2022-02-23 22:13:00 +00:00
return nil
}
func (j Job) Validate() error {
if j.Name == "" {
return fmt.Errorf("job is missing name: %w", ErrMissingField)
}
if _, err := cron.ParseStandard(j.Schedule); err != nil {
2023-05-09 21:16:08 +00:00
return fmt.Errorf("job %s has an invalid schedule: %w: %w", j.Name, err, ErrInvalidConfigValue)
2022-02-23 22:13:00 +00:00
}
if j.Config == nil {
return fmt.Errorf("job %s is missing restic config: %w", j.Name, ErrMissingField)
}
2022-02-23 22:13:00 +00:00
if err := j.Config.Validate(); err != nil {
return fmt.Errorf("job %s has invalid config: %w", j.Name, err)
}
if err := j.validateTasks(); err != nil {
return err
}
if err := j.Backup.Validate(); err != nil {
return fmt.Errorf("job %s has an invalid backup config: %w", j.Name, err)
}
2022-02-23 22:13:00 +00:00
return nil
}
2022-02-23 00:39:01 +00:00
func (j Job) AllTasks() []ExecutableTask {
allTasks := []ExecutableTask{}
// Pre tasks
2022-02-23 00:55:41 +00:00
for _, mysql := range j.MySQL {
2022-02-23 00:39:01 +00:00
allTasks = append(allTasks, mysql.GetPreTask())
}
for _, pg := range j.Postgres {
allTasks = append(allTasks, pg.GetPreTask())
}
2022-02-23 00:39:01 +00:00
for _, sqlite := range j.Sqlite {
allTasks = append(allTasks, sqlite.GetPreTask())
}
for _, jobTask := range j.Tasks {
allTasks = append(allTasks, jobTask.GetPreTasks()...)
2022-02-23 00:39:01 +00:00
}
// Add backup task
allTasks = append(allTasks, j.Backup)
2022-02-23 00:39:01 +00:00
// Post tasks
for _, jobTask := range j.Tasks {
allTasks = append(allTasks, jobTask.GetPostTasks()...)
}
2022-02-23 00:55:41 +00:00
for _, mysql := range j.MySQL {
allTasks = append(allTasks, mysql.GetPostTask())
2022-02-23 00:39:01 +00:00
}
for _, pg := range j.Postgres {
allTasks = append(allTasks, pg.GetPostTask())
}
2022-02-23 00:39:01 +00:00
for _, sqlite := range j.Sqlite {
allTasks = append(allTasks, sqlite.GetPostTask())
2022-02-23 00:39:01 +00:00
}
return allTasks
}
func (j Job) BackupPaths() []string {
paths := j.Backup.Paths
for _, t := range j.MySQL {
paths = append(paths, t.DumpToPath)
}
for _, t := range j.Postgres {
paths = append(paths, t.DumpToPath)
}
for _, t := range j.Sqlite {
paths = append(paths, t.DumpToPath)
}
return paths
}
2022-02-23 22:13:00 +00:00
func (j Job) RunBackup() error {
2022-02-23 00:39:01 +00:00
logger := GetLogger(j.Name)
restic := j.NewRestic()
if err := restic.EnsureInit(); err != nil {
return fmt.Errorf("failed to init restic for job %s: %w", j.Name, err)
}
backupPaths := j.BackupPaths()
2022-02-23 00:39:01 +00:00
for _, exTask := range j.AllTasks() {
taskCfg := TaskConfig{
BackupPaths: backupPaths,
Logger: GetChildLogger(logger, exTask.Name()),
Restic: restic,
Env: nil,
2022-02-23 00:39:01 +00:00
}
if err := exTask.RunBackup(taskCfg); err != nil {
return fmt.Errorf("failed running job %s: %w", j.Name, err)
}
}
if j.Forget != nil {
2022-02-23 22:13:00 +00:00
if err := restic.Forget(*j.Forget); err != nil {
2022-02-23 00:55:41 +00:00
return fmt.Errorf("failed forgetting and pruning job %s: %w", j.Name, err)
}
2022-02-23 00:39:01 +00:00
}
return nil
2022-02-18 22:37:51 +00:00
}
2022-02-24 07:09:04 +00:00
func (j Job) Logger() *log.Logger {
return GetLogger(j.Name)
}
func (j Job) RunRestore(snapshot string) error {
2022-02-24 07:09:04 +00:00
logger := j.Logger()
2022-02-23 22:13:00 +00:00
restic := j.NewRestic()
if _, err := restic.RunRestic("snapshots", NoOpts{}); errors.Is(err, ErrRepoNotFound) {
2022-02-23 22:13:00 +00:00
return fmt.Errorf("no repository or snapshots for job %s: %w", j.Name, err)
}
for _, exTask := range j.AllTasks() {
taskCfg := TaskConfig{
BackupPaths: nil,
Logger: GetChildLogger(logger, exTask.Name()),
Restic: restic,
Env: nil,
2022-02-23 22:13:00 +00:00
}
if backupTask, ok := exTask.(BackupFilesTask); ok {
backupTask.snapshot = snapshot
}
2022-02-23 22:13:00 +00:00
if err := exTask.RunRestore(taskCfg); err != nil {
return fmt.Errorf("failed running job %s: %w", j.Name, err)
}
}
return nil
}
func (j Job) Healthy() (bool, error) {
return j.healthy, j.lastErr
}
2022-02-24 07:09:04 +00:00
func (j Job) Run() {
2022-04-13 05:28:28 +00:00
result := JobResult{
JobName: j.Name,
JobType: "backup",
Success: true,
LastError: nil,
Message: "",
}
2022-04-13 20:44:48 +00:00
Metrics.JobStartTime.WithLabelValues(j.Name).SetToCurrentTime()
2022-02-24 07:09:04 +00:00
if err := j.RunBackup(); err != nil {
j.healthy = false
j.lastErr = err
2022-04-13 05:28:28 +00:00
j.Logger().Printf("ERROR: Backup failed: %s", err.Error())
result.Success = false
result.LastError = err
2022-02-24 07:09:04 +00:00
}
2022-04-13 05:28:28 +00:00
2022-04-13 20:44:48 +00:00
snapshots, err := j.NewRestic().ReadSnapshots()
if err != nil {
result.LastError = err
} else {
Metrics.SnapshotCurrentCount.WithLabelValues(j.Name).Set(float64(len(snapshots)))
if len(snapshots) > 0 {
latestSnapshot := snapshots[len(snapshots)-1]
Metrics.SnapshotLatestTime.WithLabelValues(j.Name).Set(float64(latestSnapshot.Time.Unix()))
}
2022-04-13 20:44:48 +00:00
}
if result.Success {
Metrics.JobFailureCount.WithLabelValues(j.Name).Set(0.0)
} else {
Metrics.JobFailureCount.WithLabelValues(j.Name).Inc()
}
2022-04-13 05:28:28 +00:00
JobComplete(result)
2022-02-24 07:09:04 +00:00
}
2022-02-24 06:53:18 +00:00
func (j Job) NewRestic() *Restic {
return &Restic{
2022-02-23 00:39:01 +00:00
Logger: GetLogger(j.Name),
Repo: j.Config.Repo,
Env: j.Config.Env,
Passphrase: j.Config.Passphrase,
GlobalOpts: j.Config.GlobalOpts,
2022-02-23 00:55:41 +00:00
Cwd: "",
2022-02-20 06:09:23 +00:00
}
}
2022-02-18 22:37:51 +00:00
type Config struct {
DefaultConfig *ResticConfig `hcl:"default_config,block"`
Jobs []Job `hcl:"job,block"`
2022-02-18 22:37:51 +00:00
}
2022-02-23 22:13:00 +00:00
func (c Config) Validate() error {
if len(c.Jobs) == 0 {
return ErrNoJobsFound
}
for _, job := range c.Jobs {
// Use default restic config if no job config is provided
// TODO: Maybe merge values here
if job.Config == nil {
job.Config = c.DefaultConfig
}
2022-02-23 22:13:00 +00:00
if err := job.Validate(); err != nil {
return err
}
}
return nil
}