From c97c514dc8066a877cc8d2e8af2695d1df247158 Mon Sep 17 00:00:00 2001 From: Lalatendu Das Date: Sun, 21 Aug 2022 20:49:02 +0000 Subject: [PATCH 1/2] pb-3031: enable the RESOURCE_CLENAUP feild to kdmp-config Added a new data RESOURCE_CLEANUP which is when set to false then CRs and other resources will not be cleaed up in case of failure. This is to help debug the issue in case of generic backup effectively Signed-off-by: Lalatendu Das --- cmd/stork/stork.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/stork/stork.go b/cmd/stork/stork.go index 0c2457427f..6c438fc6d9 100644 --- a/cmd/stork/stork.go +++ b/cmd/stork/stork.go @@ -42,6 +42,7 @@ import ( kdmpapi "github.com/portworx/kdmp/pkg/apis/kdmp/v1alpha1" "github.com/portworx/kdmp/pkg/controllers/dataexport" "github.com/portworx/kdmp/pkg/drivers" + kdmputils "github.com/portworx/kdmp/pkg/drivers/utils" "github.com/portworx/kdmp/pkg/jobratelimit" kdmpversion "github.com/portworx/kdmp/pkg/version" schedops "github.com/portworx/sched-ops/k8s/core" @@ -248,6 +249,7 @@ func run(c *cli.Context) { kdmpConfig.Data[jobratelimit.RestoreJobLimitKey] = strconv.Itoa(jobratelimit.DefaultRestoreJobLimit) kdmpConfig.Data[jobratelimit.DeleteJobLimitKey] = strconv.Itoa(jobratelimit.DefaultDeleteJobLimit) kdmpConfig.Data[jobratelimit.MaintenanceJobLimitKey] = strconv.Itoa(jobratelimit.DefaultMaintenanceJobLimit) + kdmpConfig.Data[kdmputils.ResourceCleanupKey] = kdmputils.ResourceCleanupDefaultValue // ConfigMap create failure should not fail the bring up _, err = schedops.Instance().CreateConfigMap(kdmpConfig) if err != nil && !k8s_errors.IsAlreadyExists(err) { @@ -261,7 +263,7 @@ func run(c *cli.Context) { // ConfigMap create failure should not fail the bring up _, err = schedops.Instance().CreateConfigMap(storkConfig) if err != nil && !k8s_errors.IsAlreadyExists(err) { - log.Warnf("Unable to create stork config configmap: %v", err) + log.Warnf("Unable to create stork object lock configmap: %v", err) } driverName := c.String("driver") From 53258513aa6b381a67408e4caeb69f37fcfbe007 Mon Sep 17 00:00:00 2001 From: Prashanth Kumar Date: Fri, 26 Aug 2022 14:08:19 +0000 Subject: [PATCH 2/2] vendored kdmp master --- go.mod | 2 +- go.sum | 2 + .../pkg/controllers/dataexport/reconcile.go | 46 +++++++++++++++++++ .../portworx/kdmp/pkg/drivers/utils/utils.go | 28 ++++++++++- vendor/modules.txt | 2 +- 5 files changed, 76 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 543bbf4f84..0819b01fa0 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/openshift/api v0.0.0-20210105115604-44119421ec6b github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 github.com/pborman/uuid v1.2.0 - github.com/portworx/kdmp v0.4.1-0.20220710173715-5d42efc7d149 + github.com/portworx/kdmp v0.4.1-0.20220826045120-d7260a2f9588 github.com/portworx/px-object-controller v0.0.0-20220804234424-40d3b8a84987 github.com/portworx/sched-ops v1.20.4-rc1.0.20220401024625-dbc61a336f65 github.com/portworx/torpedo v0.20.4-rc1.0.20210325154352-eb81b0cdd145 diff --git a/go.sum b/go.sum index 5991ca13cf..140acadfda 100644 --- a/go.sum +++ b/go.sum @@ -1420,6 +1420,8 @@ github.com/portworx/kdmp v0.4.1-0.20220309093511-f7b925b9e53e/go.mod h1:RAXbeaO/ github.com/portworx/kdmp v0.4.1-0.20220414053457-962507678379/go.mod h1:EAVroITfYd50a0vi/ScAILl6h5RYJteuO/pg1y3vNNw= github.com/portworx/kdmp v0.4.1-0.20220710173715-5d42efc7d149 h1:KaRMV5hWbl7raiTFo20AZaXSIBBKCadzBmrXfwU+Id0= github.com/portworx/kdmp v0.4.1-0.20220710173715-5d42efc7d149/go.mod h1:nb5AupP/63ByyqAYfZ+E32LDEnP0PjgH6w+yKXxWIgE= +github.com/portworx/kdmp v0.4.1-0.20220826045120-d7260a2f9588 h1:ccySAgCYhc4p9GbLTpCenRdO5vTogAUDCZ6q85M9Dlg= +github.com/portworx/kdmp v0.4.1-0.20220826045120-d7260a2f9588/go.mod h1:nb5AupP/63ByyqAYfZ+E32LDEnP0PjgH6w+yKXxWIgE= github.com/portworx/kvdb v0.0.0-20190105022415-cccaa09abfc9/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= github.com/portworx/kvdb v0.0.0-20200723230726-2734b7f40194/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= github.com/portworx/kvdb v0.0.0-20200929023115-b312c7519467 h1:jkqzdbOnejgSN5HG/FLt4enNrozWT/K+nlmaRm3P1II= diff --git a/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go b/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go index f588c013d4..a9d6df56da 100644 --- a/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go +++ b/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go @@ -429,6 +429,17 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er data := updateDataExportDetail{ stage: kdmpapi.DataExportStageFinal, } + // Append the job-pod log to stork's pod log in case of failure + // it is best effort approach, hence errors are ignored. + if dataExport.Status.Status == kdmpapi.DataExportStatusFailed { + if dataExport.Status.TransferID != "" { + namespace, name, err := utils.ParseJobID(dataExport.Status.TransferID) + if err != nil { + logrus.Infof("job-pod name and namespace extraction failed: %v", err) + } + appendPodLogToStork(name, namespace) + } + } cleanupTask := func() (interface{}, bool, error) { cleanupErr := c.cleanUp(driver, dataExport) if cleanupErr != nil { @@ -452,6 +463,34 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er return false, nil } +func appendPodLogToStork(jobName string, namespace string) { + // Get job and check whether it has live pod attaced to it + job, err := batch.Instance().GetJob(jobName, namespace) + if err != nil && !k8sErrors.IsNotFound(err) { + logrus.Infof("failed in getting job %v/%v with err: %v", namespace, jobName, err) + } + pods, err := core.Instance().GetPods( + job.Namespace, + map[string]string{ + "job-name": job.Name, + }, + ) + if err != nil { + logrus.Infof("failed in fetching job pods %s/%s: %v", namespace, jobName, err) + } + for _, pod := range pods.Items { + numLogLines := int64(50) + podLog, err := core.Instance().GetPodLog(pod.Name, pod.Namespace, &corev1.PodLogOptions{TailLines: &numLogLines}) + if err != nil { + logrus.Infof("error fetching log of job-pod %s: %v", pod.Name, err) + } else { + logrus.Infof("start of job-pod [%s]'s log...", pod.Name) + logrus.Infof(podLog) + logrus.Infof("end of job-pod [%s]'s log...", pod.Name) + } + } +} + func (c *Controller) createJobCredCertSecrets( dataExport *kdmpapi.DataExport, vb *kdmpapi.VolumeBackup, @@ -1190,6 +1229,13 @@ func (c *Controller) stageLocalSnapshotRestoreInProgress(ctx context.Context, da func (c *Controller) cleanUp(driver drivers.Interface, de *kdmpapi.DataExport) error { var bl *storkapi.BackupLocation + doCleanup, err := utils.DoCleanupResource() + if err != nil { + return err + } + if (de.Status.Status == kdmpapi.DataExportStatusFailed) && !doCleanup { + return nil + } if driver == nil { return fmt.Errorf("driver is nil") } diff --git a/vendor/github.com/portworx/kdmp/pkg/drivers/utils/utils.go b/vendor/github.com/portworx/kdmp/pkg/drivers/utils/utils.go index 2ba0f708ae..c6accef46c 100644 --- a/vendor/github.com/portworx/kdmp/pkg/drivers/utils/utils.go +++ b/vendor/github.com/portworx/kdmp/pkg/drivers/utils/utils.go @@ -4,9 +4,11 @@ import ( "errors" "fmt" "os" + "strconv" "strings" "github.com/aquilax/truncate" + "github.com/libopenstorage/stork/pkg/k8sutils" "github.com/portworx/kdmp/pkg/drivers" "github.com/portworx/kdmp/pkg/version" "github.com/portworx/sched-ops/k8s/apps" @@ -23,7 +25,8 @@ import ( const ( defaultPXNamespace = "kube-system" - kdmpConfig = "kdmp-config" + // KdmpConfig defines the config map name of kdmp module + KdmpConfig = "kdmp-config" // TriggeredFromStork - denotes the kopia job is triggered from stork module TriggeredFromStork = "stork" // TriggeredFromPxBackup - denotes the kopia job is triggered from px-backup module @@ -39,6 +42,10 @@ const ( ImageSecret = "image-secret" // CertSecret - cert secret prefix CertSecret = "cert-secret" + // ResourceCleanupKey - this key controls the enable or disable of the resource cleanup process. + ResourceCleanupKey = "RESOURCE_CLEANUP" + // ResourceCleanupDefaultValue is true as resource cleanup process is enabled by default for debugging user can set to false. + ResourceCleanupDefaultValue = "true" ) var ( @@ -215,13 +222,30 @@ func GetConfigValue(cm, ns, key string) string { ns, ) if err != nil { - logrus.Warnf("Failed in getting value for key [%v] from configmap[%v]", key, kdmpConfig) + logrus.Warnf("Failed in getting value for key [%v] from configmap[%v]", key, KdmpConfig) // try reading from the Env variable return os.Getenv(key) } return configMap.Data[key] } +// DoCleanupResource returns whether to cleanup the CRs & other resources. +func DoCleanupResource() (bool, error) { + doCleanup := true + cleanupResourceVal, err := k8sutils.GetConfigValue(KdmpConfig, defaultPXNamespace, ResourceCleanupKey) + if err != nil { + logrus.Errorf("Failed to get %s key from kdmp-config-map: %v", ResourceCleanupKey, err) + return true, err + } + if cleanupResourceVal != "" { + doCleanup, err = strconv.ParseBool(cleanupResourceVal) + if err != nil { + return true, err + } + } + return doCleanup, nil +} + // ResticExecutorImage returns a docker image that contains resticexecutor binary. func ResticExecutorImage() string { if customImage := strings.TrimSpace(os.Getenv(drivers.ResticExecutorImageKey)); customImage != "" { diff --git a/vendor/modules.txt b/vendor/modules.txt index d18537a7d8..54e52b5259 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -691,7 +691,7 @@ github.com/pierrec/lz4/internal/xxh32 github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.0 github.com/pmezard/go-difflib/difflib -# github.com/portworx/kdmp v0.4.1-0.20220710173715-5d42efc7d149 +# github.com/portworx/kdmp v0.4.1-0.20220826045120-d7260a2f9588 ## explicit github.com/portworx/kdmp/pkg/apis/kdmp github.com/portworx/kdmp/pkg/apis/kdmp/v1alpha1