Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add storage root folder prefix to startAfter flag #270

Merged
merged 3 commits into from
Nov 25, 2024
+7 −5
Merged

Add storage root folder prefix to startAfter flag #270

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
37c5c05
fix scan after flag
korotkov-aerospike Nov 25, 2024
c1f717c
linter warnings
korotkov-aerospike Nov 25, 2024
abc7be8
rollback config
korotkov-aerospike Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Loading
Loading status checks…
linter warnings
@korotkov-aerospike
korotkov-aerospike committed Nov 25, 2024
commit c1f717ceed220a260e9fd1e7dd82f94551e0729f
40 changes: 20 additions & 20 deletions config/config.yml
View file
Original file line number Diff line number Diff line change
@@ -81,19 +81,19 @@ backup-routines:
localStorageRemoveFiles:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns1"]
backup-policy: removeFilesPolicy
localStorage:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns2"]
backup-policy: keepFilesPolicy
localStorageNotSealed:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns2"]
backup-policy: notSealed
minioRemoveFiles:
@@ -112,7 +112,7 @@ backup-routines:
interval-cron: "@yearly"
incr-interval-cron: "*/10 * * * * *"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns5"]
set-list:
- absSet5
@@ -121,21 +121,21 @@ backup-routines:
interval-cron: "@yearly"
incr-interval-cron: "*/10 * * * * *"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: []
backup-policy: keepFilesPolicy
incrementalBackupClusterNotSealed:
interval-cron: "@yearly"
incr-interval-cron: "*/10 * * * * *"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: []
backup-policy: notSealed
incrementalBackupMultipleNSs:
interval-cron: "@yearly"
incr-interval-cron: "*/10 * * * * *"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns18", "source-ns19", "source-ns20"]
backup-policy: keepFilesPolicy
minioIncremental:
@@ -150,13 +150,13 @@ backup-routines:
edgeCases:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns7"]
backup-policy: keepFilesPolicy
filterBySetAndBin:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns8"]
set-list:
- backupSet
@@ -167,67 +167,67 @@ backup-routines:
interval-cron: "@yearly"
incr-interval-cron: "*/10 * * * * *"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns10"]
backup-policy: removeFilesPolicy
fullBackup1:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns9"]
backup-policy: keepFilesPolicy
fullBackup2:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns11"]
backup-policy: keepFilesPolicy
fullBackup3:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns12"]
backup-policy: keepFilesPolicy
fullBackupFullCluster:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: []
backup-policy: keepFilesPolicy
fullBackup3Namespaces:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns15", "source-ns16", "source-ns17"]
backup-policy: keepFilesPolicy
fullBackupSlow:
interval-cron: "*/20 * * * * *"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns13"]
backup-policy: keepFilesPolicySlow
fullBackupEncrypedCompressed128:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns18"]
backup-policy: encryptedCompressedPolicy128
fullBackupEncrypedCompressed256:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: ["source-ns21"]
backup-policy: encryptedCompressedPolicy256
noIndexesUdfsRecords:
interval-cron: "@yearly"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: [ "source-ns19" ]
backup-policy: noIndexesUdfsRecords
oneMinuteIntervalBackup:
interval-cron: "0 */1 * * * *"
source-cluster: absDefaultCluster
storage: minio
storage: local
namespaces: [ "source-ns20" ]
backup-policy: removeFilesPolicy
timestamp:
3 changes: 2 additions & 1 deletion pkg/service/storage/operations.go
View file
Original file line number Diff line number Diff line change
@@ -50,7 +50,8 @@ func ReadFiles(ctx context.Context, storage model.Storage, path string, filterSt
) ([]*bytes.Buffer, error) {
var startScanFrom string
if fromTime != nil {
startScanFrom = filepath.Join(path, strconv.FormatInt(fromTime.UnixMilli()-1, 10)) // -1 to ensure filter is greater or equal.
fromTimeStr := strconv.FormatInt(fromTime.UnixMilli()-1, 10) // -1 to ensure filter is greater or equal.
startScanFrom = filepath.Join(path, fromTimeStr)
}

reader, err := CreateReader(ctx, storage, path, false, newNameValidator(filterStr), startScanFrom)