Add postgres support for backup and restore

Dockerfile

@@ -3,9 +3,10 @@ FROM alpine:3.17
 RUN apk add --no-cache \
         bash~=5 \
         consul~=1.14 \
-        nomad~=1.4 \
         mariadb-client~=10.6 \
         mariadb-connector-c~=3.3 \
+        nomad~=1.4 \
+        postgresql15-client~=15.3 \
         rclone~=1.60 \
         redis~=7.0 \
         restic~=0.14 \

itest/bootstrap-tests.sh

@@ -8,8 +8,8 @@ echo "Hello" > /data/test.txt
 touch /data/test_database.db
 sqlite3 /data/test_database.db <<-EOF
 CREATE TABLE test_table (
-  id integer PRIMARY KEY,
-  data text NOT NULL
+  id INTEGER PRIMARY KEY,
+  data TEXT NOT NULL
 );
 
 INSERT INTO test_table(data)
@@ -22,10 +22,25 @@ until mysql --host "$MYSQL_HOST" --user "$MYSQL_USER" --password="$MYSQL_PWD" --
 done
 mysql --host "$MYSQL_HOST" --user "$MYSQL_USER" --password="$MYSQL_PWD" main <<EOF
 CREATE TABLE test_table (
-  id integer AUTO_INCREMENT PRIMARY KEY,
-  data text NOT NULL
+  id INTEGER AUTO_INCREMENT PRIMARY KEY,
+  data TEXT NOT NULL
 );
 
 INSERT INTO test_table(data)
 VALUES ("Test row");
 EOF
+
+# Create Postgres database
+export PGPASSWORD="$PGSQL_PASS"
+until psql --host "$PGSQL_HOST" --username "$PGSQL_USER" --command "SELECT datname FROM pg_database;"; do
+  sleep 1
+done
+psql -v ON_ERROR_STOP=1 --host "$PGSQL_HOST" --username "$PGSQL_USER" main <<EOF
+CREATE TABLE test_table (
+  id SERIAL PRIMARY KEY,
+  data TEXT NOT NULL
+);
+
+INSERT INTO test_table(data)
+VALUES ('Test row');
+EOF

itest/docker-compose.yml

@@ -8,6 +8,12 @@ services:
       MYSQL_ROOT_PASSWORD: shhh
       MYSQL_DATABASE: main
 
+  postgres:
+    image: postgres
+    environment:
+      POSTGRES_PASSWORD: shhh
+      POSTGRES_DB: main
+
   bootstrap:
     image: resticscheduler
     entrypoint: /bootstrap-tests.sh
@@ -15,6 +21,9 @@ services:
       MYSQL_HOST: mysql
       MYSQL_USER: root
       MYSQL_PWD: shhh
+      PGSQL_HOST: postgres
+      PGSQL_USER: postgres
+      PGSQL_PASS: shhh
     volumes:
       - ./bootstrap-tests.sh:/bootstrap-tests.sh
       - ./data:/data
@@ -25,6 +34,9 @@ services:
       MYSQL_HOST: mysql
       MYSQL_USER: root
       MYSQL_PWD: shhh
+      PGSQL_HOST: postgres
+      PGSQL_USER: postgres
+      PGSQL_PASS: shhh
     volumes:
       - ./repo:/repo
       - ./data:/data
@@ -37,6 +49,9 @@ services:
       MYSQL_HOST: mysql
       MYSQL_USER: root
       MYSQL_PWD: shhh
+      PGSQL_HOST: postgres
+      PGSQL_USER: postgres
+      PGSQL_PASS: shhh
     volumes:
       - ./validate-tests.sh:/validate-tests.sh
       - ./data:/data

itest/run.sh

@@ -10,7 +10,7 @@ rm -fr ./repo/* ./data/*
 sleep 5
 
 echo Boostrap databases and data
-docker-compose up -d mysql
+docker-compose up -d mysql postgres
 docker-compose run bootstrap
 sleep 1
 
@@ -19,9 +19,9 @@ docker-compose run main -backup IntegrationTest -once /test-backup.hcl
 
 echo Clean data
 docker-compose down -v
-docker-compose up -d mysql
+docker-compose up -d mysql postgres
 rm -fr ./data/*
-sleep 20
+sleep 15
 
 echo Run restore
 docker-compose run main -restore IntegrationTest -once /test-backup.hcl

itest/test-backup.hcl

@@ -14,6 +14,15 @@ job "IntegrationTest" {
     dump_to = "/tmp/mysql.sql"
   }
 
+  postgres "Postgres" {
+    hostname = env("PGSQL_HOST")
+    database = "main"
+    username = env("PGSQL_USER")
+    password = env("PGSQL_PASS")
+    create = true
+    dump_to = "/tmp/psql.sql"
+  }
+
   sqlite "SQLite" {
     path = "/data/test_database.db"
     dump_to = "/data/test_database.db.bak"

itest/validate-tests.sh

@@ -10,6 +10,12 @@ test -f /data/test_database.db
 sqlite3 /data/test_database.db "select data from test_table where id = 1" | grep "^Test row"
 
 # Check MySql database
-mysql --host "$MYSQL_HOST" --user "$MYSQL_USER" --password="$MYSQL_PWD" main <<-EOF | grep "^Test row"
+mysql --host "$MYSQL_HOST" --user "$MYSQL_USER" --password="$MYSQL_PWD" main <<EOF | grep "^Test row"
+select data from test_table where id = 1;
+EOF
+
+# Check Postgres database
+export PGPASSWORD="$PGSQL_PASS"
+psql --host "$PGSQL_HOST" --user "$PGSQL_USER" main <<EOF | grep "Test row"
 select data from test_table where id = 1;
 EOF

job.go

@@ -60,8 +60,9 @@ type Job struct {
 	// Meta Tasks
 	// NOTE: Now that these are also available within a task
 	// these could be removed to make task order more obvious
-	MySQL  []JobTaskMySQL  `hcl:"mysql,block"`
-	Sqlite []JobTaskSqlite `hcl:"sqlite,block"`
+	MySQL    []JobTaskMySQL    `hcl:"mysql,block"`
+	Postgres []JobTaskPostgres `hcl:"postgres,block"`
+	Sqlite   []JobTaskSqlite   `hcl:"sqlite,block"`
 
 	// Metrics and health
 	healthy bool
@@ -81,6 +82,12 @@ func (j Job) validateTasks() error {
 		}
 	}
 
+	for _, pg := range j.Postgres {
+		if err := pg.Validate(); err != nil {
+			return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
+		}
+	}
+
 	for _, sqlite := range j.Sqlite {
 		if err := sqlite.Validate(); err != nil {
 			return fmt.Errorf("job %s has an invalid task: %w", j.Name, err)
@@ -126,6 +133,10 @@ func (j Job) AllTasks() []ExecutableTask {
 		allTasks = append(allTasks, mysql.GetPreTask())
 	}
 
+	for _, pg := range j.Postgres {
+		allTasks = append(allTasks, pg.GetPreTask())
+	}
+
 	for _, sqlite := range j.Sqlite {
 		allTasks = append(allTasks, sqlite.GetPreTask())
 	}
@@ -146,6 +157,10 @@ func (j Job) AllTasks() []ExecutableTask {
 		allTasks = append(allTasks, mysql.GetPostTask())
 	}
 
+	for _, pg := range j.Postgres {
+		allTasks = append(allTasks, pg.GetPostTask())
+	}
+
 	for _, sqlite := range j.Sqlite {
 		allTasks = append(allTasks, sqlite.GetPostTask())
 	}
@@ -160,6 +175,10 @@ func (j Job) BackupPaths() []string {
 		paths = append(paths, t.DumpToPath)
 	}
 
+	for _, t := range j.Postgres {
+		paths = append(paths, t.DumpToPath)
+	}
+
 	for _, t := range j.Sqlite {
 		paths = append(paths, t.DumpToPath)
 	}

job_test.go

@@ -92,6 +92,7 @@ func TestJobValidation(t *testing.T) {
 				Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 				Forget:   nil,
 				MySQL:    []main.JobTaskMySQL{},
+				Postgres: []main.JobTaskPostgres{},
 				Sqlite:   []main.JobTaskSqlite{},
 			},
 			expectedErr: nil,
@@ -106,6 +107,7 @@ func TestJobValidation(t *testing.T) {
 				Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 				Forget:   nil,
 				MySQL:    []main.JobTaskMySQL{},
+				Postgres: []main.JobTaskPostgres{},
 				Sqlite:   []main.JobTaskSqlite{},
 			},
 			expectedErr: main.ErrMissingField,
@@ -120,6 +122,7 @@ func TestJobValidation(t *testing.T) {
 				Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 				Forget:   nil,
 				MySQL:    []main.JobTaskMySQL{},
+				Postgres: []main.JobTaskPostgres{},
 				Sqlite:   []main.JobTaskSqlite{},
 			},
 			expectedErr: main.ErrInvalidConfigValue,
@@ -134,6 +137,7 @@ func TestJobValidation(t *testing.T) {
 				Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 				Forget:   nil,
 				MySQL:    []main.JobTaskMySQL{},
+				Postgres: []main.JobTaskPostgres{},
 				Sqlite:   []main.JobTaskSqlite{},
 			},
 			expectedErr: main.ErrMutuallyExclusive,
@@ -148,6 +152,7 @@ func TestJobValidation(t *testing.T) {
 				Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 				Forget:   nil,
 				MySQL:    []main.JobTaskMySQL{},
+				Postgres: []main.JobTaskPostgres{},
 				Sqlite:   []main.JobTaskSqlite{},
 			},
 			expectedErr: main.ErrMissingField,
@@ -162,6 +167,7 @@ func TestJobValidation(t *testing.T) {
 				Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 				Forget:   nil,
 				MySQL:    []main.JobTaskMySQL{{}},
+				Postgres: []main.JobTaskPostgres{},
 				Sqlite:   []main.JobTaskSqlite{},
 			},
 			expectedErr: main.ErrMissingField,
@@ -176,6 +182,7 @@ func TestJobValidation(t *testing.T) {
 				Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 				Forget:   nil,
 				MySQL:    []main.JobTaskMySQL{},
+				Postgres: []main.JobTaskPostgres{},
 				Sqlite:   []main.JobTaskSqlite{{}},
 			},
 			expectedErr: main.ErrMissingField,
@@ -216,6 +223,7 @@ func TestConfigValidation(t *testing.T) {
 					Tasks:    []main.JobTask{},
 					Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 					MySQL:    []main.JobTaskMySQL{},
+					Postgres: []main.JobTaskPostgres{},
 					Sqlite:   []main.JobTaskSqlite{},
 				}},
 			},
@@ -232,6 +240,7 @@ func TestConfigValidation(t *testing.T) {
 					Tasks:    []main.JobTask{},
 					Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 					MySQL:    []main.JobTaskMySQL{},
+					Postgres: []main.JobTaskPostgres{},
 					Sqlite:   []main.JobTaskSqlite{},
 				}},
 			},
@@ -257,6 +266,7 @@ func TestConfigValidation(t *testing.T) {
 					Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 					Forget:   nil,
 					MySQL:    []main.JobTaskMySQL{},
+					Postgres: []main.JobTaskPostgres{},
 					Sqlite:   []main.JobTaskSqlite{},
 				}},
 			},
@@ -274,6 +284,7 @@ func TestConfigValidation(t *testing.T) {
 					Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 					Forget:   nil,
 					MySQL:    []main.JobTaskMySQL{},
+					Postgres: []main.JobTaskPostgres{},
 					Sqlite:   []main.JobTaskSqlite{},
 				}}},
 			expectedErr: main.ErrMissingField,

main_test.go

@@ -52,6 +52,7 @@ func TestRunJobs(t *testing.T) {
 		Backup:   main.BackupFilesTask{Paths: []string{"/test"}}, //nolint:exhaustruct
 		Forget:   nil,
 		MySQL:    []main.JobTaskMySQL{},
+		Postgres: []main.JobTaskPostgres{},
 		Sqlite:   []main.JobTaskSqlite{},
 	}
 

tasks.go

@@ -67,7 +67,7 @@ func (t *JobTaskScript) SetName(name string) {
 	t.name = name
 }
 
-// JobTaskMySQL is a sqlite backup task that performs required pre and post tasks.
+// JobTaskMySQL is a MySQL backup task that performs required pre and post tasks.
 type JobTaskMySQL struct {
 	Port          int      `hcl:"port,optional"`
 	Name          string   `hcl:"name,label"`
@@ -187,6 +187,144 @@ func (t JobTaskMySQL) GetPostTask() ExecutableTask {
 	}
 }
 
+// JobTaskPostgres is a postgres backup task that performs required pre and post tasks.
+type JobTaskPostgres struct {
+	Port          int      `hcl:"port,optional"`
+	Name          string   `hcl:"name,label"`
+	Hostname      string   `hcl:"hostname,optional"`
+	Database      string   `hcl:"database,optional"`
+	Username      string   `hcl:"username,optional"`
+	Password      string   `hcl:"password,optional"`
+	Tables        []string `hcl:"tables,optional"`
+	DumpToPath    string   `hcl:"dump_to"`
+	NoTablespaces bool     `hcl:"no_tablespaces,optional"`
+	Clean         bool     `hcl:"clean,optional"`
+	Create        bool     `hcl:"create,optional"`
+}
+
+func (t JobTaskPostgres) Paths() []string {
+	return []string{t.DumpToPath}
+}
+
+func (t JobTaskPostgres) Validate() error {
+	if t.DumpToPath == "" {
+		return fmt.Errorf("task %s is missing dump_to path: %w", t.Name, ErrMissingField)
+	}
+
+	if stat, err := os.Stat(t.DumpToPath); err != nil {
+		if !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf(
+				"task %s: invalid dump_to: could not stat path: %s: %w",
+				t.Name,
+				t.DumpToPath,
+				ErrInvalidConfigValue,
+			)
+		}
+	} else if stat.Mode().IsDir() {
+		return fmt.Errorf("task %s: dump_to cannot be a directory: %w", t.Name, ErrInvalidConfigValue)
+	}
+
+	if len(t.Tables) > 0 && t.Database == "" {
+		return fmt.Errorf(
+			"task %s is invalid. Must specify a database to use tables: %w",
+			t.Name,
+			ErrMissingField,
+		)
+	}
+
+	return nil
+}
+
+//nolint:cyclop
+func (t JobTaskPostgres) GetPreTask() ExecutableTask {
+	command := []string{"pg_dump"}
+	if t.Database == "" {
+		command = []string{"pg_dumpall"}
+	}
+
+	command = append(command, "--file", t.DumpToPath)
+
+	if t.Hostname != "" {
+		command = append(command, "--host", t.Hostname)
+	}
+
+	if t.Port != 0 {
+		command = append(command, "--port", fmt.Sprintf("%d", t.Port))
+	}
+
+	if t.Username != "" {
+		command = append(command, "--username", t.Username)
+	}
+
+	if t.NoTablespaces {
+		command = append(command, "--no-tablespaces")
+	}
+
+	if t.Clean {
+		command = append(command, "--clean")
+	}
+
+	if t.Create {
+		command = append(command, "--create")
+	}
+
+	for _, table := range t.Tables {
+		command = append(command, "--table", table)
+	}
+
+	if t.Database != "" {
+		command = append(command, t.Database)
+	}
+
+	env := map[string]string{}
+	if t.Password != "" {
+		env["PGPASSWORD"] = t.Password
+	}
+
+	return JobTaskScript{
+		name:      t.Name,
+		env:       env,
+		Cwd:       ".",
+		OnBackup:  strings.Join(command, " "),
+		OnRestore: "",
+	}
+}
+
+func (t JobTaskPostgres) GetPostTask() ExecutableTask {
+	command := []string{"psql"}
+
+	if t.Hostname != "" {
+		command = append(command, "--host", t.Hostname)
+	}
+
+	if t.Port != 0 {
+		command = append(command, "--port", fmt.Sprintf("%d", t.Port))
+	}
+
+	if t.Username != "" {
+		command = append(command, "--username", t.Username)
+	}
+
+	if t.Database != "" {
+		command = append(command, t.Database)
+	}
+
+	command = append(command, "<", t.DumpToPath)
+
+	env := map[string]string{}
+	if t.Password != "" {
+		env["PGPASSWORD"] = t.Password
+	}
+
+	return JobTaskScript{
+		name:      t.Name,
+		env:       env,
+		Cwd:       ".",
+		OnBackup:  "",
+		OnRestore: strings.Join(command, " "),
+	}
+}
+
 // JobTaskSqlite is a sqlite backup task that performs required pre and post tasks.
 type JobTaskSqlite struct {
 	Name       string `hcl:"name,label"`
@@ -299,11 +437,12 @@ func (t *BackupFilesTask) Validate() error {
 
 // JobTask represents a single task within a backup job.
 type JobTask struct {
-	Name        string          `hcl:"name,label"`
-	PreScripts  []JobTaskScript `hcl:"pre_script,block"`
-	PostScripts []JobTaskScript `hcl:"post_script,block"`
-	MySQL       []JobTaskMySQL  `hcl:"mysql,block"`
-	Sqlite      []JobTaskSqlite `hcl:"sqlite,block"`
+	Name        string            `hcl:"name,label"`
+	PreScripts  []JobTaskScript   `hcl:"pre_script,block"`
+	PostScripts []JobTaskScript   `hcl:"post_script,block"`
+	MySQL       []JobTaskMySQL    `hcl:"mysql,block"`
+	Postgres    []JobTaskPostgres `hcl:"postgres,block"`
+	Sqlite      []JobTaskSqlite   `hcl:"sqlite,block"`
 }
 
 func (t JobTask) Validate() error {

tasks_test.go

@@ -165,6 +165,28 @@ func TestJobTaskSql(t *testing.T) {
 			preRestore:  "",
 			postRestore: "mysql --host host --port 3306 --user user --password=pass db < ./simple.sql",
 		},
+		{
+			name: "psql all",
+			task: main.JobTaskPostgres{
+				Name:          "simple",
+				Hostname:      "host",
+				Port:          6543,
+				Username:      "user",
+				Password:      "pass",
+				Database:      "db",
+				NoTablespaces: true,
+				Create:        true,
+				Clean:         true,
+				Tables:        []string{"table1", "table2"},
+				DumpToPath:    "./simple.sql",
+			},
+			validationErr: nil,
+			preBackup: "pg_dump --file ./simple.sql --host host --port 6543 --username user --no-tablespaces" +
+				" --clean --create --table table1 --table table2 db",
+			postBackup:  "",
+			preRestore:  "",
+			postRestore: "psql --host host --port 6543 --username user db < ./simple.sql",
+		},
 		// Sqlite
 		{
 			name: "sqlite simple",