From 532cb2cfc9b390f3cd1890af9fc53182710fbc93 Mon Sep 17 00:00:00 2001 From: Nicolas Thauvin Date: Sat, 5 Jul 2025 22:12:45 +0200 Subject: [PATCH 1/5] Rename var and fix dumpSettings zero time Use a fileTime instead of uniformTimestamp to store the creation time of files. Fix dumpSettings() to use the current time when the file time is zero. --- main.go | 36 ++++++++++++++++++++---------------- pg_back.conf | 2 +- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/main.go b/main.go index a54f7f1..f1010e5 100644 --- a/main.go +++ b/main.go @@ -304,9 +304,9 @@ func run() (retVal error) { defer db.Close() // Generate a single datetime that will be used in all files generated by pg_back - var uniformTimestamp time.Time + var fileTime time.Time if opts.UniformTimestamp { - uniformTimestamp = time.Now() + fileTime = time.Now() } if !opts.DumpOnly { @@ -322,7 +322,7 @@ func run() (retVal error) { } else { l.Infoln("dumping globals without role passwords") } - if err := dumpGlobals(opts.Directory, opts.Mode, opts.TimeFormat, dumpRolePasswords, conninfo, producedFiles, uniformTimestamp); err != nil { + if err := dumpGlobals(opts.Directory, opts.Mode, opts.TimeFormat, dumpRolePasswords, conninfo, producedFiles, fileTime); err != nil { return fmt.Errorf("pg_dumpall of globals failed: %w", err) } @@ -332,7 +332,7 @@ func run() (retVal error) { perr *pgPrivError ) - if err := dumpSettings(opts.Directory, opts.Mode, opts.TimeFormat, db, producedFiles, uniformTimestamp); err != nil { + if err := dumpSettings(opts.Directory, opts.Mode, opts.TimeFormat, db, producedFiles, fileTime); err != nil { if errors.As(err, &verr) || errors.As(err, &perr) { l.Warnln(err) } else { @@ -340,7 +340,7 @@ func run() (retVal error) { } } - if err := dumpConfigFiles(opts.Directory, opts.Mode, opts.TimeFormat, db, producedFiles, uniformTimestamp); err != nil { + if err := dumpConfigFiles(opts.Directory, opts.Mode, opts.TimeFormat, db, producedFiles, fileTime); err != nil { return fmt.Errorf("could not dump configuration files: %w", err) } } @@ -392,7 +392,7 @@ func run() (retVal error) { CipherPassphrase: passphrase, CipherPublicKey: publicKey, EncryptKeepSrc: opts.EncryptKeepSrc, - When: uniformTimestamp, + When: fileTime, ExitCode: -1, PgDumpVersion: pgDumpVersion, } @@ -944,7 +944,7 @@ func pgToolVersion(tool string) int { return numver } -func dumpGlobals(dir string, mode int, timeFormat string, withRolePasswords bool, conninfo *ConnInfo, fc chan<- sumFileJob, uniformTimestamp time.Time) error { +func dumpGlobals(dir string, mode int, timeFormat string, withRolePasswords bool, conninfo *ConnInfo, fc chan<- sumFileJob, when time.Time) error { command := execPath("pg_dumpall") args := []string{"-g", "-w"} @@ -976,11 +976,11 @@ func dumpGlobals(dir string, mode int, timeFormat string, withRolePasswords bool args = append(args, "--no-role-passwords") } - if uniformTimestamp.IsZero() { - uniformTimestamp = time.Now() + if when.IsZero() { + when = time.Now() } - file := formatDumpPath(dir, timeFormat, "sql", "pg_globals", uniformTimestamp, 0) + file := formatDumpPath(dir, timeFormat, "sql", "pg_globals", when, 0) args = append(args, "-f", file) if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { @@ -1021,8 +1021,12 @@ func dumpGlobals(dir string, mode int, timeFormat string, withRolePasswords bool return nil } -func dumpSettings(dir string, mode int, timeFormat string, db *pg, fc chan<- sumFileJob, uniformTimestamp time.Time) error { - file := formatDumpPath(dir, timeFormat, "out", "pg_settings", uniformTimestamp, 0) +func dumpSettings(dir string, mode int, timeFormat string, db *pg, fc chan<- sumFileJob, when time.Time) error { + if when.IsZero() { + when = time.Now() + } + + file := formatDumpPath(dir, timeFormat, "out", "pg_settings", when, 0) if err := os.MkdirAll(filepath.Dir(file), 0o700); err != nil { return err @@ -1056,12 +1060,12 @@ func dumpSettings(dir string, mode int, timeFormat string, db *pg, fc chan<- sum return nil } -func dumpConfigFiles(dir string, mode int, timeFormat string, db *pg, fc chan<- sumFileJob, uniformTimestamp time.Time) error { +func dumpConfigFiles(dir string, mode int, timeFormat string, db *pg, fc chan<- sumFileJob, when time.Time) error { for _, param := range []string{"hba_file", "ident_file"} { - if uniformTimestamp.IsZero() { - uniformTimestamp = time.Now() + if when.IsZero() { + when = time.Now() } - file := formatDumpPath(dir, timeFormat, "out", param, uniformTimestamp, 0) + file := formatDumpPath(dir, timeFormat, "out", param, when, 0) if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { return err diff --git a/pg_back.conf b/pg_back.conf index a016097..d99ff70 100644 --- a/pg_back.conf +++ b/pg_back.conf @@ -53,7 +53,7 @@ with_templates = false # Dump only databases, excluding configuration and globals dump_only = false -# Apply the same consistent timestamp to all filenames generated +# Apply the same timestamp to all filenames generated # by pg_back instead of using individual file creation times uniform_timestamp = false From 8e311926c69d6c988980500b5d81fe5af76fe114 Mon Sep 17 00:00:00 2001 From: Nicolas Thauvin Date: Sat, 5 Jul 2025 22:21:30 +0200 Subject: [PATCH 2/5] Reorganize usage --- config.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/config.go b/config.go index 1cb0b90..57031b4 100644 --- a/config.go +++ b/config.go @@ -280,11 +280,11 @@ func parseCli(args []string) (options, []string, error) { pflag.PrintDefaults() } + pflag.StringVarP(&opts.CfgFile, "config", "c", defaultCfgFile, "alternate config file") pflag.BoolVar(&opts.NoConfigFile, "no-config-file", false, "skip reading config file\n") pflag.StringVarP(&opts.BinDirectory, "bin-directory", "B", "", "PostgreSQL binaries directory. Empty to search $PATH") pflag.StringVarP(&opts.Directory, "backup-directory", "b", "/var/backups/postgresql", "store dump files there") pflag.StringVarP(&mode, "backup-file-mode", "m", "0600", "mode to apply to dump files") - pflag.StringVarP(&opts.CfgFile, "config", "c", defaultCfgFile, "alternate config file") pflag.StringSliceVarP(&opts.ExcludeDbs, "exclude-dbs", "D", []string{}, "list of databases to exclude") pflag.BoolVarP(&opts.WithTemplates, "with-templates", "t", false, "include templates") WithoutTemplates := pflag.Bool("without-templates", false, "force exclude templates") @@ -297,31 +297,31 @@ func parseCli(args []string) (options, []string, error) { pflag.IntVarP(&opts.DirJobs, "parallel-backup-jobs", "J", 1, "number of parallel jobs to dumps when using directory format") pflag.IntVarP(&opts.CompressLevel, "compress", "Z", -1, "compression level for compressed formats") pflag.StringVarP(&opts.SumAlgo, "checksum-algo", "S", "none", "signature algorithm: none sha1 sha224 sha256 sha384 sha512") + pflag.BoolVar(&opts.UniformTimestamp, "uniform-timestamp", false, "Use the same timestamp for all pg_back files instead of individual\ncreation times") pflag.StringVarP(&purgeInterval, "purge-older-than", "P", "30", "purge backups older than this duration in days\nuse an interval with units \"s\" (seconds), \"m\" (minutes) or \"h\" (hours)\nfor less than a day.") pflag.StringVarP(&purgeKeep, "purge-min-keep", "K", "0", "minimum number of dumps to keep when purging or 'all' to keep\neverything") pflag.StringVar(&opts.PreHook, "pre-backup-hook", "", "command to run before taking dumps") pflag.StringVar(&opts.PostHook, "post-backup-hook", "", "command to run after taking dumps\n") - pflag.BoolVar(&opts.UniformTimestamp, "uniform-timestamp", false, "Use the same timestamp for all pg_back files instead of individual creation times") pflag.BoolVar(&opts.Encrypt, "encrypt", false, "encrypt the dumps") NoEncrypt := pflag.Bool("no-encrypt", false, "do not encrypt the dumps") pflag.BoolVar(&opts.EncryptKeepSrc, "encrypt-keep-src", false, "keep original files when encrypting") NoEncryptKeepSrc := pflag.Bool("no-encrypt-keep-src", false, "do not keep original files when encrypting") pflag.BoolVar(&opts.Decrypt, "decrypt", false, "decrypt files in the backup directory instead of dumping. DBNAMEs become\nglobs to select files") - pflag.StringVar(&opts.CipherPassphrase, "cipher-pass", "", "cipher passphrase for encryption and decryption\n") - pflag.StringVar(&opts.CipherPublicKey, "cipher-public-key", "", "AGE public key for encryption; in Bech32 encoding starting with 'age1'\n") - pflag.StringVar(&opts.CipherPrivateKey, "cipher-private-key", "", "AGE private key for decryption; in Bech32 encoding starting with 'AGE-SECRET-KEY-1'\n") + pflag.StringVar(&opts.CipherPassphrase, "cipher-pass", "", "cipher passphrase for encryption and decryption") + pflag.StringVar(&opts.CipherPublicKey, "cipher-public-key", "", "AGE public key for encryption; in Bech32 encoding starting with 'age1'") + pflag.StringVar(&opts.CipherPrivateKey, "cipher-private-key", "", "AGE private key for decryption; in Bech32 encoding starting with\n'AGE-SECRET-KEY-1'\n") pflag.StringVar(&opts.Upload, "upload", "none", "upload produced files to target (s3, gcs,..) use \"none\" to override\nconfiguration file and disable upload") pflag.StringVar(&opts.UploadPrefix, "upload-prefix", "", "add this prefix to uploaded files, similar to a target directory") deleteUploaded := pflag.String("delete-uploaded", "no", "delete local file after upload") pflag.StringVar(&opts.Download, "download", "none", "download files from target (s3, gcs,..) instead of dumping. DBNAMEs become\nglobs to select files") - pflag.StringVar(&opts.ListRemote, "list-remote", "none", "list the remote files on s3, gcs, sftp, azure instead of dumping. DBNAMEs become\nglobs to select files") + pflag.StringVar(&opts.ListRemote, "list-remote", "none", "list the remote files on s3, gcs, sftp, azure instead of dumping. DBNAMEs\nbecome globs to select files") purgeRemote := pflag.String("purge-remote", "no", "purge the file on remote location after upload, with the same rules\nas the local directory") - pflag.StringVar(&opts.B2Bucket, "b2-bucket", "", "B2 bucket") - pflag.StringVar(&opts.B2KeyID, "b2-key-id", "", "B2 access key ID") - pflag.StringVar(&opts.B2AppKey, "b2-app-key", "", "B2 app key") + pflag.StringVar(&opts.B2Bucket, "b2-bucket", "", "Backblaze B2 bucket") + pflag.StringVar(&opts.B2KeyID, "b2-key-id", "", "Backblaze B2 access key ID") + pflag.StringVar(&opts.B2AppKey, "b2-app-key", "", "Backblaze B2 app key") B2ForcePath := pflag.String("b2-force-path", "no", "force path style addressing instead of virtual hosted bucket\naddressing") B2ConcurrentConnections := pflag.Int("b2-concurrent-connections", 5, "set the amount of concurrent b2 http connections") From a164fcaa81b458e31dfb6be53117e15eb630f9c8 Mon Sep 17 00:00:00 2001 From: Nicolas Thauvin Date: Sat, 5 Jul 2025 22:21:58 +0200 Subject: [PATCH 3/5] Update test fixture for newer versions of PostgreSQL --- testdata/fixture.sql | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/testdata/fixture.sql b/testdata/fixture.sql index 58ae9e6..3d787fa 100644 --- a/testdata/fixture.sql +++ b/testdata/fixture.sql @@ -13,15 +13,16 @@ GRANT CONNECT ON DATABASE b2 TO u2; \c b1 SET ROLE u1; -CREATE TABLE t1 AS SELECT generate_series(0, 9) i; -CREATE TABLE t2 AS SELECT generate_series(10, 19) j; -CREATE TABLE t3 AS SELECT generate_series(0, 9) i; -CREATE TABLE t4 AS SELECT generate_series(10, 19) j; +CREATE TABLE IF NOT EXISTS t1 AS SELECT generate_series(0, 9) i; +CREATE TABLE IF NOT EXISTS t2 AS SELECT generate_series(10, 19) j; +CREATE TABLE IF NOT EXISTS t3 AS SELECT generate_series(0, 9) i; +CREATE TABLE IF NOT EXISTS t4 AS SELECT generate_series(10, 19) j; \c b2 +GRANT CREATE,USAGE ON SCHEMA public TO u2; SET ROLE u2; -CREATE TABLE t1 AS SELECT generate_series(0, 9) i; -CREATE TABLE t2 AS SELECT generate_series(10, 19) j; -CREATE TABLE t3 AS SELECT generate_series(0, 9) i; -CREATE TABLE t4 AS SELECT generate_series(10, 19) j; +CREATE TABLE IF NOT EXISTS t1 AS SELECT generate_series(0, 9) i; +CREATE TABLE IF NOT EXISTS t2 AS SELECT generate_series(10, 19) j; +CREATE TABLE IF NOT EXISTS t3 AS SELECT generate_series(0, 9) i; +CREATE TABLE IF NOT EXISTS t4 AS SELECT generate_series(10, 19) j; From d40ad9f2547d4b89f545c5614c2e5baf8a5966c6 Mon Sep 17 00:00:00 2001 From: Nicolas Thauvin Date: Sat, 5 Jul 2025 22:22:39 +0200 Subject: [PATCH 4/5] Update docs changelog and contributors --- CHANGELOG.md | 4 ++++ CONTRIBUTORS | 2 ++ README.md | 24 ++++++++++++++++++------ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e495ffb..96c227a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## pg_back 2.6.0 * Allow to override the connection user per database +* Add the --delete-uploaded to delete local file after upload +* Add the --backup-file-mode option to configure the permissions on dumps +* Add the --uniform-timestamp option to create all files of a run with the same + time in the filenames ## pg_back 2.5.0 diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 7a42828..7bc5074 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -16,3 +16,5 @@ Massimo Lusetti Kenny Root Pierrick Chovelon Dennis Urban +Julian Vanden Broeck +Stéphane Klein diff --git a/README.md b/README.md index 7540450..7d2a72a 100644 --- a/README.md +++ b/README.md @@ -66,12 +66,6 @@ If default and command line options are not enough, a configuration file may be provided with `-c ` (see [pg_back.conf](pg_back.conf)). (Note: see below to convert configuration files from version 1.) -If the default output directory `/var/backups/postgresql` does not exist or has -improper ownership for your user, use `-b` to give the path where to store the -files. The path may contain the `{dbname}` keyword, that would be replaced by -the name of the database being dumped, this permits to dump each database in -its own directory. - To connect to PostgreSQL, use the `-h`, `-p`, `-U` and `-d` options. If you need less known connection options such as `sslcert` and `sslkey`, you can give a `keyword=value` libpq connection string like `pg_dump` and `pg_dumpall` @@ -83,6 +77,24 @@ The other command line options let you tweak what is dumped, purged, and how it is done. These options can be put in a configuration file. The command line options override configuration options. +### Output + +If the default output directory `/var/backups/postgresql` does not exist or has +improper ownership for your user, use `-b` to give the path where to store the +files. The path may contain the `{dbname}` keyword, that would be replaced by +the name of the database being dumped, this permits to dump each database in +its own directory. + +By default the files are created with script file permissions, 0600, it can be +changed with the `--backup-file-mode` option. Permissions on directories are +deduced from the file mode to add the `x` bit if necessary. + +A run of pg_back creates multiple files, with the date and time of their +creation in their filename. While this help finding at what time the data would +be restored, all timestamps in filename are potentially different. The +`--uniform-timestamp` changes this behavious to use the same date and time for +all files output by a run, it is the time of the first dump of the run. + ### Per-database configuration Per-database configuration can only be done with a configuration file. The From 0b6d67a98abcfe33fdad3e4b4649c94214805438 Mon Sep 17 00:00:00 2001 From: Nicolas Thauvin Date: Sat, 5 Jul 2025 22:48:20 +0200 Subject: [PATCH 5/5] Fix --delete-uploaded cli option and document it It did not overwrite the default or value of the config file --- README.md | 6 ++++++ config.go | 12 +++++++----- main.go | 2 ++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7d2a72a..0022018 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,12 @@ the prefix is separated by a / in the remote location. The `--purge-remote` option can be set to `yes` to apply the same purge policy on the remote location as the local directory. +The `--delete-uploaded` option can be set to `yes` (or `no` to override the +configuration file value) to remove a local file as soon as it is uploaded +successfully. (Note: it will leave unencrypted local files when +`--encrypt-keep-src` and `--encrypt` are active because only encrypted files +are uploaded in that case) + When files are encrypted and their unencrypted source is kept, only encrypted files are uploaded. diff --git a/config.go b/config.go index 57031b4..4b8de54 100644 --- a/config.go +++ b/config.go @@ -576,7 +576,7 @@ gkLoop: } subs := cfg.Sections() - knonw_perdb := []string{ + known_perdb := []string{ "format", "parallel_backup_jobs", "compress_level", "checksum_algorithm", "purge_older_than", "purge_min_keep", "schemas", "exclude_schemas", "tables", "exclude_tables", "pg_dump_options", "with_blobs", "user", @@ -589,7 +589,7 @@ gkLoop: dbkLoop: for _, v := range sub.KeyStrings() { - for _, c := range knonw_perdb { + for _, c := range known_perdb { if v == c { continue dbkLoop } @@ -886,6 +886,9 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin for _, dbo := range opts.PerDbOpts { dbo.SumAlgo = cliOpts.SumAlgo } + case "uniform-timestamp": + opts.UniformTimestamp = cliOpts.UniformTimestamp + case "purge-older-than": opts.PurgeInterval = cliOpts.PurgeInterval for _, dbo := range opts.PerDbOpts { @@ -923,6 +926,8 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin opts.ListRemote = cliOpts.ListRemote case "purge-remote": opts.PurgeRemote = cliOpts.PurgeRemote + case "delete-uploaded": + opts.DeleteUploaded = cliOpts.DeleteUploaded case "b2-bucket": opts.B2Bucket = cliOpts.B2Bucket @@ -991,10 +996,7 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin opts.Username = cliOpts.Username case "dbname": opts.ConnDb = cliOpts.ConnDb - case "uniform-timestamp": - opts.UniformTimestamp = cliOpts.UniformTimestamp } } - return opts } diff --git a/main.go b/main.go index f1010e5..de663b6 100644 --- a/main.go +++ b/main.go @@ -1615,6 +1615,8 @@ func postProcessFiles(inFiles chan sumFileJob, wg *sync.WaitGroup, opts options) } continue } + + l.Infoln("uploaded", j.Path) if opts.DeleteUploaded { l.Infoln("removing", j.Path) os.Remove(j.Path)