From ec20260bf7f36bbbe7cb54bd33661dcdd40e46a4 Mon Sep 17 00:00:00 2001 From: woblerr Date: Wed, 4 Dec 2024 01:34:01 +0300 Subject: [PATCH] Add support for deleting backups using the --after-timestamp option. --- COMMANDS.md | 19 ++++++++++-- cmd/backup_clean.go | 58 ++++++++++++++++++++++++++++------- cmd/constants.go | 3 ++ gpbckpconfig/utils_db.go | 18 ++++++++++- gpbckpconfig/utils_db_test.go | 12 ++++++++ 5 files changed, 96 insertions(+), 14 deletions(-) diff --git a/COMMANDS.md b/COMMANDS.md index c0c3da1..b6797ac 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -3,6 +3,7 @@ - [Delete all backups from local storage older than the specified time condition](#delete-all-backups-from-local-storage-older-than-the-specified-time-condition) - [Delete all backups using storage plugin older than n days](#delete-all-backups-using-storage-plugin-older-than-n-days) - [Delete all backups using storage plugin older than timestamp](#delete-all-backups-using-storage-plugin-older-than-timestamp) + - [Delete all backups using storage plugin newer than timestamp](#delete-all-backups-using-storage-plugin-newer-than-timestamp) - [Using container](#using-container) - [Delete a specific existing backup (`backup-delete`)](#delete-a-specific-existing-backup-backup-delete) - [Examples](#examples-1) @@ -34,8 +35,9 @@ Available options for `backup-clean` command and their description: Delete all existing backups older than the specified time condition. To delete backup sets older than the given timestamp, use the --before-timestamp option. -To delete backup sets older than the given number of days, use the --older-than-day option. -Only --older-than-days or --before-timestamp option must be specified, not both. +To delete backup sets older than the given number of days, use the --older-than-day option. +To delete backup sets newer than the given timestamp, use the --after-timestamp option. +Only --older-than-days, --before-timestamp or --after-timestamp option must be specified. By default, the existence of dependent backups is checked and deletion process is not performed, unless the --cascade option is passed in. @@ -67,6 +69,7 @@ Usage: gpbackman backup-clean [flags] Flags: + --after-timestamp string delete backup sets newer than the given timestamp --backup-dir string the full path to backup directory for local backups --before-timestamp string delete backup sets older than the given timestamp --cascade delete all dependent backups @@ -117,6 +120,18 @@ Delete all backups older than timestamp `20240101100000` and all dependent backu --cascade ``` +### Delete all backups using storage plugin newer than timestamp + +Delete all backups newer than timestamp `20240101100000` and all dependent backups: +```bash +./gpbackman backup-clean \ + --after-timestamp 20240101100000 \ + --plugin-config /tmp/gpbackup_plugin_config.yaml \ + --cascade +``` + +Be careful, using the flag may lead to the deletion of actual backups. Backups newer than the specified timestamp are deleted. For the example above, `20240101220000`, `20240102100000`, etc. will be deleted. + ## Using container Delete all backups using `gpbackup_s3_plugin` storage plugin older than 7 days: diff --git a/cmd/backup_clean.go b/cmd/backup_clean.go index 160f8e3..77b9e31 100644 --- a/cmd/backup_clean.go +++ b/cmd/backup_clean.go @@ -15,6 +15,7 @@ import ( // Flags for the gpbackman backup-clean command (backupCleanCmd) var ( backupCleanBeforeTimestamp string + backupCleanAfterTimestamp string backupCleanPluginConfigFile string backupCleanBackupDir string backupCleanOlderThenDays uint @@ -28,8 +29,9 @@ var backupCleanCmd = &cobra.Command{ Long: `Delete all existing backups older than the specified time condition. To delete backup sets older than the given timestamp, use the --before-timestamp option. -To delete backup sets older than the given number of days, use the --older-than-day option. -Only --older-than-days or --before-timestamp option must be specified, not both. +To delete backup sets older than the given number of days, use the --older-than-day option. +To delete backup sets newer than the given timestamp, use the --after-timestamp option. +Only --older-than-days, --before-timestamp or --after-timestamp option must be specified. By default, the existence of dependent backups is checked and deletion process is not performed, unless the --cascade option is passed in. @@ -90,6 +92,12 @@ func init() { "", "delete backup sets older than the given timestamp", ) + backupCleanCmd.PersistentFlags().StringVar( + &backupCleanAfterTimestamp, + afterTimestampFlagName, + "", + "delete backup sets newer than the given timestamp", + ) backupCleanCmd.PersistentFlags().StringVar( &backupCleanBackupDir, backupDirFlagName, @@ -102,7 +110,7 @@ func init() { 1, "the number of parallel processes to delete local backups", ) - backupCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThenDaysFlagName) + backupCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThenDaysFlagName, afterTimestampFlagName) } // These flag checks are applied only for backup-clean command. @@ -120,6 +128,15 @@ func doCleanBackupFlagValidation(flags *pflag.FlagSet) { if flags.Changed(olderThenDaysFlagName) { beforeTimestamp = gpbckpconfig.GetTimestampOlderThen(backupCleanOlderThenDays) } + // If after-timestamp flag is specified and have correct values. + if flags.Changed(afterTimestampFlagName) { + err = gpbckpconfig.CheckTimestamp(backupCleanAfterTimestamp) + if err != nil { + gplog.Error(textmsg.ErrorTextUnableValidateFlag(backupCleanAfterTimestamp, afterTimestampFlagName, err)) + execOSExit(exitErrorCode) + } + afterTimestamp = backupCleanAfterTimestamp + } // backup-dir anf plugin-config flags cannot be used together. err = checkCompatibleFlags(flags, backupDirFlagName, pluginConfigFileFlagName) if err != nil { @@ -153,8 +170,8 @@ func doCleanBackupFlagValidation(flags *pflag.FlagSet) { execOSExit(exitErrorCode) } } - if beforeTimestamp == "" { - gplog.Error(textmsg.ErrorTextUnableValidateValue(textmsg.ErrorValidationValue(), olderThenDaysFlagName, beforeTimestampFlagName)) + if beforeTimestamp == "" && afterTimestamp == "" { + gplog.Error(textmsg.ErrorTextUnableValidateValue(textmsg.ErrorValidationValue(), olderThenDaysFlagName, beforeTimestampFlagName, afterTimestampFlagName)) execOSExit(exitErrorCode) } } @@ -185,12 +202,12 @@ func cleanBackup() error { gplog.Error(textmsg.ErrorTextUnableReadPluginConfigFile(err)) return err } - err = backupCleanDBPlugin(backupCleanCascade, beforeTimestamp, backupCleanPluginConfigFile, pluginConfig, hDB) + err = backupCleanDBPlugin(backupCleanCascade, beforeTimestamp, afterTimestamp, backupCleanPluginConfigFile, pluginConfig, hDB) if err != nil { return err } } else { - err := backupCleanDBLocal(backupCleanCascade, beforeTimestamp, backupCleanBackupDir, backupCleanParallelProcesses, hDB) + err := backupCleanDBLocal(backupCleanCascade, beforeTimestamp, afterTimestamp, backupCleanBackupDir, backupCleanParallelProcesses, hDB) if err != nil { return err } @@ -198,8 +215,8 @@ func cleanBackup() error { return nil } -func backupCleanDBPlugin(deleteCascade bool, cutOffTimestamp, pluginConfigPath string, pluginConfig *utils.PluginConfig, hDB *sql.DB) error { - backupList, err := gpbckpconfig.GetBackupNamesBeforeTimestamp(cutOffTimestamp, hDB) +func backupCleanDBPlugin(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestamp, pluginConfigPath string, pluginConfig *utils.PluginConfig, hDB *sql.DB) error { + backupList, err := fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, hDB) if err != nil { gplog.Error(textmsg.ErrorTextUnableReadHistoryDB(err)) return err @@ -219,8 +236,8 @@ func backupCleanDBPlugin(deleteCascade bool, cutOffTimestamp, pluginConfigPath s return nil } -func backupCleanDBLocal(deleteCascade bool, cutOffTimestamp, backupDir string, maxParallelProcesses int, hDB *sql.DB) error { - backupList, err := gpbckpconfig.GetBackupNamesBeforeTimestamp(cutOffTimestamp, hDB) +func backupCleanDBLocal(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestamp, backupDir string, maxParallelProcesses int, hDB *sql.DB) error { + backupList, err := fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, hDB) if err != nil { gplog.Error(textmsg.ErrorTextUnableReadHistoryDB(err)) return err @@ -236,3 +253,22 @@ func backupCleanDBLocal(deleteCascade bool, cutOffTimestamp, backupDir string, m } return nil } + +// Get the list of backup names for deletion. +func fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp string, hDB *sql.DB) ([]string, error) { + var backupList []string + var err error + if cutOffTimestamp != "" { + backupList, err = gpbckpconfig.GetBackupNamesBeforeTimestamp(cutOffTimestamp, hDB) + if err != nil { + return nil, err + } + } + if cutOffAfterTimestamp != "" { + backupList, err = gpbckpconfig.GetBackupNamesAfterTimestamp(cutOffAfterTimestamp, hDB) + if err != nil { + return nil, err + } + } + return backupList, nil +} diff --git a/cmd/constants.go b/cmd/constants.go index d4734df..5af66bc 100644 --- a/cmd/constants.go +++ b/cmd/constants.go @@ -32,6 +32,7 @@ const ( forceFlagName = "force" olderThenDaysFlagName = "older-than-days" beforeTimestampFlagName = "before-timestamp" + afterTimestampFlagName = "after-timestamp" typeFlagName = "type" tableFlagName = "table" schemaFlagName = "schema" @@ -53,4 +54,6 @@ const ( var ( // Timestamp to delete all backups before. beforeTimestamp string + // Timestamp to delete all backups after. + afterTimestamp string ) diff --git a/gpbckpconfig/utils_db.go b/gpbckpconfig/utils_db.go index be16eb1..f5f5750 100644 --- a/gpbckpconfig/utils_db.go +++ b/gpbckpconfig/utils_db.go @@ -39,6 +39,10 @@ func GetBackupNamesBeforeTimestamp(timestamp string, historyDB *sql.DB) ([]strin return execQueryFunc(getBackupNameBeforeTimestampQuery(timestamp), historyDB) } +func GetBackupNamesAfterTimestamp(timestamp string, historyDB *sql.DB) ([]string, error) { + return execQueryFunc(getBackupNameAfterTimestampQuery(timestamp), historyDB) +} + func GetBackupNamesForCleanBeforeTimestamp(timestamp string, historyDB *sql.DB) ([]string, error) { return execQueryFunc(getBackupNameForCleanBeforeTimestampQuery(timestamp), historyDB) } @@ -73,7 +77,7 @@ ORDER BY timestamp DESC; `, backupName, backupName) } -// Only active backups, "In progress", deleted and failed statuses - hidden. +// Only active backups, "In progress", deleted and failed statuses - hidden. func getBackupNameBeforeTimestampQuery(timestamp string) string { return fmt.Sprintf(` SELECT timestamp @@ -85,6 +89,18 @@ ORDER BY timestamp DESC; `, timestamp, BackupStatusInProgress, DateDeletedPluginFailed, DateDeletedLocalFailed) } +// Only active backups, "In progress", deleted and failed statuses - hidden. +func getBackupNameAfterTimestampQuery(timestamp string) string { + return fmt.Sprintf(` +SELECT timestamp +FROM backups +WHERE timestamp > '%s' + AND status != '%s' + AND date_deleted IN ('', '%s', '%s') +ORDER BY timestamp DESC; +`, timestamp, BackupStatusInProgress, DateDeletedPluginFailed, DateDeletedLocalFailed) +} + // Only deleted backups. func getBackupNameForCleanBeforeTimestampQuery(timestamp string) string { return fmt.Sprintf(` diff --git a/gpbckpconfig/utils_db_test.go b/gpbckpconfig/utils_db_test.go index f0abaa9..47363b7 100644 --- a/gpbckpconfig/utils_db_test.go +++ b/gpbckpconfig/utils_db_test.go @@ -72,6 +72,18 @@ WHERE timestamp < '20240101120000' AND status != 'In Progress' AND date_deleted IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed') ORDER BY timestamp DESC; +`}, + { + name: "Test getBackupNameAfterTimestampQuery", + value: "20240101120000", + function: getBackupNameAfterTimestampQuery, + want: ` +SELECT timestamp +FROM backups +WHERE timestamp > '20240101120000' + AND status != 'In Progress' + AND date_deleted IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed') +ORDER BY timestamp DESC; `}, } for _, tt := range tests {