Skip to content

Commit

Permalink
Extend Etcd CRD and etcd-druid to Support Configurable Delta Snap…
Browse files Browse the repository at this point in the history 
…shot Retention (#651)

* Extend Etcd CRD to include `DeltaSnapshotRetentionPeriod` field

* feat: Add DeltaSnapshotRetentionPeriod flag propagation

Updated etcd-druid to propagate --delta-snapshot-retention-period flag from the Etcd CRD to etcd-backup-restore.
  • Loading branch information
@seshachalam-yv
seshachalam-yv authored Aug 9, 2023
1 parent 7dd3afe commit 8df7251
Show file tree
Hide file tree
Showing 7 changed files with 122 additions and 41 deletions.
7 changes: 7 additions & 0 deletions api/v1alpha1/types_etcd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,13 @@ type BackupSpec struct {
// DeltaSnapshotMemoryLimit defines the memory limit after which delta snapshots will be taken
// +optional
DeltaSnapshotMemoryLimit *resource.Quantity `json:"deltaSnapshotMemoryLimit,omitempty"`
// DeltaSnapshotRetentionPeriod defines the duration for which delta snapshots will be retained, excluding the latest snapshot set.
// The value should be a string formatted as a duration (e.g., '1s', '2m', '3h', '4d')
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Pattern="^([0-9][0-9]*([.][0-9]+)?(s|m|h|d))+$"
// +optional
DeltaSnapshotRetentionPeriod *metav1.Duration `json:"deltaSnapshotRetentionPeriod,omitempty"`

// SnapshotCompression defines the specification for compression of Snapshots.
// +optional
SnapshotCompression *CompressionSpec `json:"compression,omitempty"`
Expand Down
5 changes: 5 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions charts/druid/charts/crds/templates/10-crd-druid.gardener.cloud_etcds.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,13 @@ spec:
description: DeltaSnapshotPeriod defines the period after which
delta snapshots will be taken
type: string
deltaSnapshotRetentionPeriod:
description: DeltaSnapshotRetentionPeriod defines the duration
for which delta snapshots will be retained, excluding the latest
snapshot set. The value should be a string formatted as a duration
(e.g., '1s', '2m', '3h', '4d')
pattern: ^([0-9][0-9]*([.][0-9]+)?(s|m|h|d))+$
type: string
enableProfiling:
description: EnableProfiling defines if profiling should be enabled
for the etcd-backup-restore-sidecar
Expand Down
7 changes: 7 additions & 0 deletions config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,13 @@ spec:
description: DeltaSnapshotPeriod defines the period after which
delta snapshots will be taken
type: string
deltaSnapshotRetentionPeriod:
description: DeltaSnapshotRetentionPeriod defines the duration
for which delta snapshots will be retained, excluding the latest
snapshot set. The value should be a string formatted as a duration
(e.g., '1s', '2m', '3h', '4d')
pattern: ^([0-9][0-9]*([.][0-9]+)?(s|m|h|d))+$
type: string
enableProfiling:
description: EnableProfiling defines if profiling should be enabled
for the etcd-backup-restore-sidecar
Expand Down
125 changes: 87 additions & 38 deletions pkg/component/etcd/statefulset/statefulset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,26 @@ var _ = Describe("Statefulset", func() {
Expect(metav1.HasAnnotation(sts.ObjectMeta, "gardener.cloud/scaled-to-multi-node")).To(BeFalse())
})
})

Context("DeltaSnapshotRetentionPeriod field is set in Etcd CRD", func() {
It("should include --delta-snapshot-retention-period flag in etcd-backup-restore container command", func() {
etcd.Spec.Backup.DeltaSnapshotRetentionPeriod = &metav1.Duration{Duration: time.Hour * 24}
values = GenerateValues(
etcd,
pointer.Int32(clientPort),
pointer.Int32(serverPort),
pointer.Int32(backupPort),
imageEtcd,
imageBR,
checkSumAnnotations, false, true)
stsDeployer = New(cl, logr.Discard(), values)
Expect(stsDeployer.Deploy(ctx)).To(Succeed())

sts := &appsv1.StatefulSet{}
Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), sts)).To(Succeed())
checkStatefulset(sts, values)
})
})
})

Context("when statefulset exists", func() {
Expand Down Expand Up @@ -252,6 +272,26 @@ var _ = Describe("Statefulset", func() {
Expect(updatedSts.Spec.PodManagementPolicy).To(Equal(appsv1.ParallelPodManagement))
})
})

Context("DeltaSnapshotRetentionPeriod field is updated in Etcd CRD", func() {
It("should update --delta-snapshot-retention-period flag in etcd-backup-restore container command", func() {
etcd.Spec.Backup.DeltaSnapshotRetentionPeriod = &metav1.Duration{Duration: time.Hour * 48}
values = GenerateValues(
etcd,
pointer.Int32(clientPort),
pointer.Int32(serverPort),
pointer.Int32(backupPort),
imageEtcd,
imageBR,
checkSumAnnotations, false, true)
stsDeployer = New(cl, logr.Discard(), values)
Expect(stsDeployer.Deploy(ctx)).To(Succeed())

sts := &appsv1.StatefulSet{}
Expect(cl.Get(ctx, kutil.Key(namespace, values.Name), sts)).To(Succeed())
checkStatefulset(sts, values)
})
})
})

Context("with backup", func() {
Expand Down Expand Up @@ -421,8 +461,6 @@ func checkBackup(etcd *druidv1alpha1.Etcd, sts *appsv1.StatefulSet) {

func checkStatefulset(sts *appsv1.StatefulSet, values Values) {
checkStsOwnerRefs(sts.ObjectMeta.OwnerReferences, values)
store, err := druidutils.StorageProviderFromInfraProvider(values.BackupStore.Provider)
Expect(err).NotTo(HaveOccurred())
Expect(*sts).To(MatchFields(IgnoreExtras, Fields{
"ObjectMeta": MatchFields(IgnoreExtras, Fields{
"Name": Equal(values.Name),
Expand Down Expand Up @@ -538,42 +576,7 @@ func checkStatefulset(sts *appsv1.StatefulSet, values Values) {
}),

backupRestore: MatchFields(IgnoreExtras, Fields{
"Args": MatchAllElements(cmdIterator, Elements{
"server": Equal("server"),
"--cert=/var/etcd/ssl/client/client/tls.crt": Equal("--cert=/var/etcd/ssl/client/client/tls.crt"),
"--key=/var/etcd/ssl/client/client/tls.key": Equal("--key=/var/etcd/ssl/client/client/tls.key"),
"--cacert=/var/etcd/ssl/client/ca/ca.crt": Equal("--cacert=/var/etcd/ssl/client/ca/ca.crt"),
"--server-cert=/var/etcd/ssl/client/server/tls.crt": Equal("--server-cert=/var/etcd/ssl/client/server/tls.crt"),
"--server-key=/var/etcd/ssl/client/server/tls.key": Equal("--server-key=/var/etcd/ssl/client/server/tls.key"),
"--data-dir=/var/etcd/data/new.etcd": Equal("--data-dir=/var/etcd/data/new.etcd"),
"--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp": Equal("--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp"),
"--insecure-transport=false": Equal("--insecure-transport=false"),
"--insecure-skip-tls-verify=false": Equal("--insecure-skip-tls-verify=false"),
"--snapstore-temp-directory=/var/etcd/data/temp": Equal("--snapstore-temp-directory=/var/etcd/data/temp"),
fmt.Sprintf("%s=%s", "--etcd-connection-timeout-leader-election", etcdLeaderElectionConnectionTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-connection-timeout-leader-election", values.LeaderElection.EtcdConnectionTimeout.Duration.String())),
"--etcd-connection-timeout=5m": Equal("--etcd-connection-timeout=5m"),
"--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"),
"--enable-member-lease-renewal=true": Equal("--enable-member-lease-renewal=true"),
"--k8s-heartbeat-duration=10s": Equal("--k8s-heartbeat-duration=10s"),
fmt.Sprintf("--defragmentation-schedule=%s", *values.DefragmentationSchedule): Equal(fmt.Sprintf("--defragmentation-schedule=%s", *values.DefragmentationSchedule)),
fmt.Sprintf("--schedule=%s", *values.FullSnapshotSchedule): Equal(fmt.Sprintf("--schedule=%s", *values.FullSnapshotSchedule)),
fmt.Sprintf("%s=%s", "--garbage-collection-policy", *values.GarbageCollectionPolicy): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-policy", *values.GarbageCollectionPolicy)),
fmt.Sprintf("%s=%s", "--storage-provider", store): Equal(fmt.Sprintf("%s=%s", "--storage-provider", store)),
fmt.Sprintf("%s=%s", "--store-prefix", values.BackupStore.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", values.BackupStore.Prefix)),
fmt.Sprintf("--delta-snapshot-memory-limit=%d", values.DeltaSnapshotMemoryLimit.Value()): Equal(fmt.Sprintf("--delta-snapshot-memory-limit=%d", values.DeltaSnapshotMemoryLimit.Value())),
fmt.Sprintf("--garbage-collection-policy=%s", *values.GarbageCollectionPolicy): Equal(fmt.Sprintf("--garbage-collection-policy=%s", *values.GarbageCollectionPolicy)),
fmt.Sprintf("--endpoints=https://%s-local:%d", values.Name, clientPort): Equal(fmt.Sprintf("--endpoints=https://%s-local:%d", values.Name, clientPort)),
fmt.Sprintf("--service-endpoints=https://%s:%d", values.ClientServiceName, clientPort): Equal(fmt.Sprintf("--service-endpoints=https://%s:%d", values.ClientServiceName, clientPort)),
fmt.Sprintf("--embedded-etcd-quota-bytes=%d", int64(values.Quota.Value())): Equal(fmt.Sprintf("--embedded-etcd-quota-bytes=%d", int64(values.Quota.Value()))),
fmt.Sprintf("%s=%s", "--delta-snapshot-period", values.DeltaSnapshotPeriod.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-period", values.DeltaSnapshotPeriod.Duration.String())),
fmt.Sprintf("%s=%s", "--garbage-collection-period", values.GarbageCollectionPeriod.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-period", values.GarbageCollectionPeriod.Duration.String())),
fmt.Sprintf("%s=%s", "--auto-compaction-mode", *values.AutoCompactionMode): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-mode", *values.AutoCompactionMode)),
fmt.Sprintf("%s=%s", "--auto-compaction-retention", *values.AutoCompactionRetention): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-retention", *values.AutoCompactionRetention)),
fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", values.EtcdSnapshotTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", values.EtcdSnapshotTimeout.Duration.String())),
fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", values.EtcdDefragTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", values.EtcdDefragTimeout.Duration.String())),
fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", values.DeltaSnapLeaseName): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", values.DeltaSnapLeaseName)),
fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", values.FullSnapLeaseName): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", values.FullSnapLeaseName)),
}),
"Args": MatchAllElements(cmdIterator, expectedBackupArgs(&values)),
"Ports": ConsistOf([]corev1.ContainerPort{
{
Name: "server",
Expand Down Expand Up @@ -930,3 +933,49 @@ func checkLocalProviderVaues(etcd *druidv1alpha1.Etcd, sts *appsv1.StatefulSet,
MountPath: "/home/nonroot/" + container,
}))
}

func expectedBackupArgs(values *Values) Elements {
store, err := druidutils.StorageProviderFromInfraProvider(values.BackupStore.Provider)
Expect(err).NotTo(HaveOccurred())
elements := Elements{
"server": Equal("server"),
"--cert=/var/etcd/ssl/client/client/tls.crt": Equal("--cert=/var/etcd/ssl/client/client/tls.crt"),
"--key=/var/etcd/ssl/client/client/tls.key": Equal("--key=/var/etcd/ssl/client/client/tls.key"),
"--cacert=/var/etcd/ssl/client/ca/ca.crt": Equal("--cacert=/var/etcd/ssl/client/ca/ca.crt"),
"--server-cert=/var/etcd/ssl/client/server/tls.crt": Equal("--server-cert=/var/etcd/ssl/client/server/tls.crt"),
"--server-key=/var/etcd/ssl/client/server/tls.key": Equal("--server-key=/var/etcd/ssl/client/server/tls.key"),
"--data-dir=/var/etcd/data/new.etcd": Equal("--data-dir=/var/etcd/data/new.etcd"),
"--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp": Equal("--restoration-temp-snapshots-dir=/var/etcd/data/restoration.temp"),
"--insecure-transport=false": Equal("--insecure-transport=false"),
"--insecure-skip-tls-verify=false": Equal("--insecure-skip-tls-verify=false"),
"--snapstore-temp-directory=/var/etcd/data/temp": Equal("--snapstore-temp-directory=/var/etcd/data/temp"),
fmt.Sprintf("%s=%s", "--etcd-connection-timeout-leader-election", etcdLeaderElectionConnectionTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-connection-timeout-leader-election", values.LeaderElection.EtcdConnectionTimeout.Duration.String())),
"--etcd-connection-timeout=5m": Equal("--etcd-connection-timeout=5m"),
"--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"),
"--enable-member-lease-renewal=true": Equal("--enable-member-lease-renewal=true"),
"--k8s-heartbeat-duration=10s": Equal("--k8s-heartbeat-duration=10s"),
fmt.Sprintf("--defragmentation-schedule=%s", *values.DefragmentationSchedule): Equal(fmt.Sprintf("--defragmentation-schedule=%s", *values.DefragmentationSchedule)),
fmt.Sprintf("--schedule=%s", *values.FullSnapshotSchedule): Equal(fmt.Sprintf("--schedule=%s", *values.FullSnapshotSchedule)),
fmt.Sprintf("%s=%s", "--garbage-collection-policy", *values.GarbageCollectionPolicy): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-policy", *values.GarbageCollectionPolicy)),
fmt.Sprintf("%s=%s", "--storage-provider", store): Equal(fmt.Sprintf("%s=%s", "--storage-provider", store)),
fmt.Sprintf("%s=%s", "--store-prefix", values.BackupStore.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", values.BackupStore.Prefix)),
fmt.Sprintf("--delta-snapshot-memory-limit=%d", values.DeltaSnapshotMemoryLimit.Value()): Equal(fmt.Sprintf("--delta-snapshot-memory-limit=%d", values.DeltaSnapshotMemoryLimit.Value())),
fmt.Sprintf("--garbage-collection-policy=%s", *values.GarbageCollectionPolicy): Equal(fmt.Sprintf("--garbage-collection-policy=%s", *values.GarbageCollectionPolicy)),
fmt.Sprintf("--endpoints=https://%s-local:%d", values.Name, clientPort): Equal(fmt.Sprintf("--endpoints=https://%s-local:%d", values.Name, clientPort)),
fmt.Sprintf("--service-endpoints=https://%s:%d", values.ClientServiceName, clientPort): Equal(fmt.Sprintf("--service-endpoints=https://%s:%d", values.ClientServiceName, clientPort)),
fmt.Sprintf("--embedded-etcd-quota-bytes=%d", int64(values.Quota.Value())): Equal(fmt.Sprintf("--embedded-etcd-quota-bytes=%d", int64(values.Quota.Value()))),
fmt.Sprintf("%s=%s", "--delta-snapshot-period", values.DeltaSnapshotPeriod.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-period", values.DeltaSnapshotPeriod.Duration.String())),
fmt.Sprintf("%s=%s", "--garbage-collection-period", values.GarbageCollectionPeriod.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-period", values.GarbageCollectionPeriod.Duration.String())),
fmt.Sprintf("%s=%s", "--auto-compaction-mode", *values.AutoCompactionMode): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-mode", *values.AutoCompactionMode)),
fmt.Sprintf("%s=%s", "--auto-compaction-retention", *values.AutoCompactionRetention): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-retention", *values.AutoCompactionRetention)),
fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", values.EtcdSnapshotTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", values.EtcdSnapshotTimeout.Duration.String())),
fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", values.EtcdDefragTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", values.EtcdDefragTimeout.Duration.String())),
fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", values.DeltaSnapLeaseName): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", values.DeltaSnapLeaseName)),
fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", values.FullSnapLeaseName): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", values.FullSnapLeaseName)),
}

if values.DeltaSnapshotRetentionPeriod != nil {
elements[fmt.Sprintf("--delta-snapshot-retention-period=%s", values.DeltaSnapshotRetentionPeriod.Duration.String())] = Equal(fmt.Sprintf("--delta-snapshot-retention-period=%s", values.DeltaSnapshotRetentionPeriod.Duration.String()))
}
return elements
}
3 changes: 2 additions & 1 deletion pkg/component/etcd/statefulset/values.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ type Values struct {

EnableProfiling *bool

DeltaSnapshotPeriod *metav1.Duration
DeltaSnapshotPeriod *metav1.Duration
DeltaSnapshotRetentionPeriod *metav1.Duration

SnapshotCompression *druidv1alpha1.CompressionSpec
HeartbeatDuration *metav1.Duration
Expand Down
9 changes: 7 additions & 2 deletions pkg/component/etcd/statefulset/values_helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,9 @@ func GenerateValues(
BackupStore: etcd.Spec.Backup.Store,
EnableProfiling: etcd.Spec.Backup.EnableProfiling,

DeltaSnapshotPeriod: etcd.Spec.Backup.DeltaSnapshotPeriod,
DeltaSnapshotMemoryLimit: etcd.Spec.Backup.DeltaSnapshotMemoryLimit,
DeltaSnapshotPeriod: etcd.Spec.Backup.DeltaSnapshotPeriod,
DeltaSnapshotRetentionPeriod: etcd.Spec.Backup.DeltaSnapshotRetentionPeriod,
DeltaSnapshotMemoryLimit: etcd.Spec.Backup.DeltaSnapshotMemoryLimit,

DefragmentationSchedule: etcd.Spec.Etcd.DefragmentationSchedule,
FullSnapshotSchedule: etcd.Spec.Backup.FullSnapshotSchedule,
Expand Down Expand Up @@ -277,6 +278,10 @@ func getBackupRestoreCommand(val Values) []string {
command = append(command, "--delta-snapshot-period="+val.DeltaSnapshotPeriod.Duration.String())
}

if val.DeltaSnapshotRetentionPeriod != nil {
command = append(command, "--delta-snapshot-retention-period="+val.DeltaSnapshotRetentionPeriod.Duration.String())
}

var deltaSnapshotMemoryLimit = defaultSnapshotMemoryLimit
if val.DeltaSnapshotMemoryLimit != nil {
deltaSnapshotMemoryLimit = val.DeltaSnapshotMemoryLimit.Value()
Expand Down

0 comments on commit 8df7251

Please sign in to comment.