From 29f495ef07e2991ac6d904aaf2148cb204a14167 Mon Sep 17 00:00:00 2001 From: shkumari-px Date: Tue, 3 Oct 2023 05:56:26 +0000 Subject: [PATCH] vendor changes from kdmp master to stork master --- go.mod | 4 +- go.sum | 4 +- .../pkg/controllers/dataexport/reconcile.go | 30 +- .../pkg/drivers/kopiabackup/kopiabackup.go | 43 +-- .../drivers/kopiabackup/kopiabackuplive.go | 257 ------------------ .../portworx/kdmp/pkg/executor/common.go | 6 +- vendor/modules.txt | 6 +- 7 files changed, 33 insertions(+), 317 deletions(-) delete mode 100644 vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackuplive.go diff --git a/go.mod b/go.mod index fa4721e1e8..86387cc2b5 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( ) require ( - github.com/portworx/kdmp v0.4.1-0.20230830195819-704706dda5c7 + github.com/portworx/kdmp v0.4.1-0.20230927104323-84564ac8b457 k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 ) @@ -324,7 +324,7 @@ replace ( github.com/libopenstorage/autopilot-api => github.com/libopenstorage/autopilot-api v0.6.1-0.20210301232050-ca2633c6e114 github.com/libopenstorage/openstorage => github.com/libopenstorage/openstorage v0.0.0-20230511212757-41751b27d69f github.com/libopenstorage/secrets => github.com/libopenstorage/secrets v0.0.0-20220413195519-57d1c446c5e9 - github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20230830195819-704706dda5c7 + github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20230927104323-84564ac8b457 github.com/portworx/sched-ops => github.com/portworx/sched-ops v1.20.4-rc1.0.20230922202555-69648f749f5a github.com/portworx/torpedo => github.com/portworx/torpedo v0.0.0-20230927164041-818bdade92ff gopkg.in/fsnotify.v1 v1.4.7 => github.com/fsnotify/fsnotify v1.4.7 diff --git a/go.sum b/go.sum index 5d6ad62b0a..39f7462158 100644 --- a/go.sum +++ b/go.sum @@ -2749,8 +2749,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polyfloyd/go-errorlint v1.0.5/go.mod h1:APVvOesVSAnne5SClsPxPdfvZTVDojXh1/G3qb5wjGI= github.com/portworx/dcos-secrets v0.0.0-20180616013705-8e8ec3f66611/go.mod h1:4hklRW/4DQpLqkcXcjtNprbH2tz/sJaNtqinfPWl/LA= -github.com/portworx/kdmp v0.4.1-0.20230830195819-704706dda5c7 h1:SuTAPkl16RvspsGameIaW63zPqBv9i30X0Yv6NEWklM= -github.com/portworx/kdmp v0.4.1-0.20230830195819-704706dda5c7/go.mod h1:KFeJuczRm1Jw6AdejFLcXHB6yrjfKY/X1brfWDmXgr8= +github.com/portworx/kdmp v0.4.1-0.20230927104323-84564ac8b457 h1:mNaAaeHjKvm20V/4yOLspmhEtOjssWeyCf1110lRvqQ= +github.com/portworx/kdmp v0.4.1-0.20230927104323-84564ac8b457/go.mod h1:+p1H1FdhDSpjwzhAqSppzMAnd2DJjVVcnDcL2hTvwyY= github.com/portworx/kvdb v0.0.0-20190105022415-cccaa09abfc9/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= github.com/portworx/kvdb v0.0.0-20200723230726-2734b7f40194/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= github.com/portworx/kvdb v0.0.0-20200929023115-b312c7519467/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= diff --git a/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go b/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go index 590c94330a..c7e2bca09e 100644 --- a/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go +++ b/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go @@ -466,7 +466,7 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er var cleanupErr error // Need to retain the old reason present in the dataexport CR, so passing the reason again. data := updateDataExportDetail{ - stage: kdmpapi.DataExportStageFinal, + stage: kdmpapi.DataExportStageFinal, reason: dataExport.Status.Reason, } // Append the job-pod log to stork's pod log in case of failure @@ -520,6 +520,13 @@ func appendPodLogToStork(jobName string, namespace string) { } for _, pod := range pods.Items { numLogLines := int64(50) + podDescribe, err := core.Instance().GetPodByName(pod.Name, pod.Namespace) + if err != nil { + logrus.Infof("Error fetching description of job-pod[%s] :%v", pod.Name, err) + } + logrus.Infof("start of job-pod [%s]'s description", pod.Name) + logrus.Infof("Describe %v", podDescribe) + logrus.Infof("end of job-pod [%s]'s description", pod.Name) podLog, err := core.Instance().GetPodLog(pod.Name, pod.Namespace, &corev1.PodLogOptions{TailLines: &numLogLines}) if err != nil { logrus.Infof("error fetching log of job-pod %s: %v", pod.Name, err) @@ -545,27 +552,6 @@ func (c *Controller) createJobCredCertSecrets( blNamespace = vb.Spec.BackupLocation.Namespace } if driverName == drivers.KopiaBackup { - pods, err := core.Instance().GetPodsUsingPVC(srcPVCName, dataExport.Spec.Source.Namespace) - if err != nil { - msg := fmt.Sprintf("error fetching pods using PVC %s/%s: %v", dataExport.Spec.Source.Namespace, srcPVCName, err) - logrus.Errorf(msg) - data := updateDataExportDetail{ - status: kdmpapi.DataExportStatusFailed, - reason: msg, - } - return data, err - } - // filter out the pods that are create by us - count := len(pods) - for _, pod := range pods { - labels := pod.ObjectMeta.Labels - if _, ok := labels[drivers.DriverNameLabel]; ok { - count-- - } - } - if count > 0 { - namespace = utils.AdminNamespace - } blName = dataExport.Spec.Destination.Name blNamespace = dataExport.Spec.Destination.Namespace } diff --git a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go index 8e96e9cb21..800201cb38 100644 --- a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go +++ b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go @@ -262,6 +262,7 @@ func jobFor( jobOption drivers.JobOpts, jobName string, resources corev1.ResourceRequirements, + nodeName string, ) (*batchv1.Job, error) { backupName := jobName @@ -375,6 +376,10 @@ func jobFor( }, } + if len(nodeName) != 0 { + job.Spec.Template.Spec.NodeName = nodeName + } + // Add the image secret in job spec only if it is present in the stork deployment. if len(imageRegistrySecret) != 0 { job.Spec.Template.Spec.ImagePullSecrets = utils.ToImagePullSecret(utils.GetImageSecretName(jobName)) @@ -462,49 +467,31 @@ func buildJob(jobName string, jobOptions drivers.JobOpts) (*batchv1.Job, error) } var resourceNamespace string var live bool + var nodeName string // filter out the pods that are create by us - count := len(pods) for _, pod := range pods { labels := pod.ObjectMeta.Labels if _, ok := labels[drivers.DriverNameLabel]; ok { - count-- + continue + } + if pod.Status.Phase == "Running" { + // get the nodeName, if the pods is in Running state, So that we can schedule + // kopia job on the same node. + nodeName = pod.Spec.NodeName + break } } - if count > 0 { - resourceNamespace = utils.AdminNamespace - live = true - } else { - resourceNamespace = jobOptions.Namespace - live = false - } + resourceNamespace = jobOptions.Namespace if err := utils.SetupServiceAccount(jobName, resourceNamespace, roleFor(live)); err != nil { errMsg := fmt.Sprintf("error creating service account %s/%s: %v", resourceNamespace, jobName, err) logrus.Errorf("%s: %v", fn, errMsg) return nil, fmt.Errorf(errMsg) } - // run a "live" backup if a pvc is mounted (mount a kubelet directory with pod volumes) - if live { - logrus.Debugf("buildJob: pod %v phase %v pvc: %v/%v", pods[0].Name, pods[0].Status.Phase, resourceNamespace, jobOptions.SourcePVCName) - if pods[0].Status.Phase == corev1.PodPending { - errMsg := fmt.Sprintf("pods %v is using pvc %v/%v but it is in pending state, backup is not possible", pods[0].Name, resourceNamespace, jobOptions.SourcePVCName) - logrus.Errorf("%s: %v", fn, errMsg) - return nil, fmt.Errorf(errMsg) - } - pvcNamespace := jobOptions.Namespace - jobOptions.Namespace = resourceNamespace - return jobForLiveBackup( - jobOptions, - jobName, - pods[0], - resources, - pvcNamespace, - ) - } - return jobFor( jobOptions, jobName, resources, + nodeName, ) } diff --git a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackuplive.go b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackuplive.go deleted file mode 100644 index 42c2756dd3..0000000000 --- a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackuplive.go +++ /dev/null @@ -1,257 +0,0 @@ -package kopiabackup - -import ( - "fmt" - "strings" - - "github.com/portworx/kdmp/pkg/drivers" - "github.com/portworx/kdmp/pkg/drivers/utils" - coreops "github.com/portworx/sched-ops/k8s/core" - "github.com/sirupsen/logrus" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8shelper "k8s.io/component-helpers/storage/volume" -) - -var ( - defaultPodsMountPath = "/var/lib/kubelet/pods" -) - -func jobForLiveBackup( - jobOption drivers.JobOpts, - jobName string, - mountPod corev1.Pod, - resources corev1.ResourceRequirements, - pvcNamespace string, -) (*batchv1.Job, error) { - volDir, err := getVolumeDirectory(jobOption.SourcePVCName, jobOption.SourcePVCNamespace) - if err != nil { - return nil, err - } - // pod volumes reside under /var/lib/kubelet/pods//volumes// directory. - // mount /var/lib/kubelet/pods//volumes as a /data directory to a resticexecutor job and - // use /data/*/ as a backup directory and determine volume plugin by resticexecutor. - var podVolumesPath string - if len(jobOption.PodDataPath) == 0 { - podVolumesPath = fmt.Sprintf("%s/%s/volumes", defaultPodsMountPath, mountPod.UID) - } else { - logrus.Debugf("selecting pod data path %v from config map", jobOption.PodDataPath) - podVolumesPath = fmt.Sprintf("%s/%s/volumes", jobOption.PodDataPath, mountPod.UID) - } - backupPath := fmt.Sprintf("/data/*/%s", volDir) - - backupName := jobName - - labels := addJobLabels(jobOption.Labels) - cmd := strings.Join([]string{ - "/kopiaexecutor", - "backup", - "--volume-backup-name", - backupName, - "--credentials", - jobOption.DataExportName, - "--backup-location", - jobOption.BackupLocationName, - "--backup-location-namespace", - jobOption.BackupLocationNamespace, - "--backup-namespace", - jobOption.Namespace, - "--repository", - toRepoName(jobOption.RepoPVCName, pvcNamespace), - "--source-path-glob", - backupPath, - }, " ") - - if jobOption.Compression != "" { - splitCmd := strings.Split(cmd, " ") - splitCmd = append(splitCmd, "--compression", jobOption.Compression) - cmd = strings.Join(splitCmd, " ") - } - - privileged := true - kopiaExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.KopiaExecutorImage, - jobOption.KopiaImageExecutorSource, - jobOption.KopiaImageExecutorSourceNs, - jobName, - jobOption) - if err != nil { - errMsg := fmt.Errorf("failed to get the executor image details for job %s", jobName) - logrus.Errorf("%v", errMsg) - return nil, errMsg - } - tolerations, err := utils.GetTolerationsFromDeployment(jobOption.KopiaImageExecutorSource, - jobOption.KopiaImageExecutorSourceNs) - if err != nil { - logrus.Errorf("failed to get the toleration details: %v", err) - return nil, fmt.Errorf("failed to get the toleration details for job [%s/%s]", jobOption.Namespace, jobName) - } - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: jobName, - Namespace: jobOption.Namespace, - Annotations: map[string]string{ - utils.SkipResourceAnnotation: "true", - }, - Labels: labels, - }, - Spec: batchv1.JobSpec{ - BackoffLimit: &utils.JobPodBackOffLimit, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyOnFailure, - ServiceAccountName: jobName, - NodeName: mountPod.Spec.NodeName, - Containers: []corev1.Container{ - { - Name: "kopiaexecutor", - Image: kopiaExecutorImage, - ImagePullPolicy: corev1.PullAlways, - Command: []string{ - "/bin/sh", - "-x", - "-c", - cmd, - }, - Resources: resources, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "vol", - MountPath: "/data", - }, - { - Name: "cred-secret", - MountPath: drivers.KopiaCredSecretMount, - ReadOnly: true, - }, - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: &privileged, - }, - }, - }, - Tolerations: tolerations, - Volumes: []corev1.Volume{ - { - Name: "vol", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: podVolumesPath, - }, - }, - }, - { - Name: "cred-secret", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: utils.GetCredSecretName(jobOption.DataExportName), - }, - }, - }, - }, - }, - }, - }, - } - - // Add the image secret in job spec only if it is present in the stork deployment. - if len(imageRegistrySecret) != 0 { - job.Spec.Template.Spec.ImagePullSecrets = utils.ToImagePullSecret(utils.GetImageSecretName(jobName)) - } - - if len(jobOption.NfsServer) != 0 { - volumeMount := corev1.VolumeMount{ - Name: utils.NfsVolumeName, - MountPath: drivers.NfsMount, - } - job.Spec.Template.Spec.Containers[0].VolumeMounts = append( - job.Spec.Template.Spec.Containers[0].VolumeMounts, - volumeMount, - ) - volume := corev1.Volume{ - Name: utils.NfsVolumeName, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: utils.GetPvcNameForJob(jobName), - }, - }, - } - job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, volume) - } - - if drivers.CertFilePath != "" { - volumeMount := corev1.VolumeMount{ - Name: utils.TLSCertMountVol, - MountPath: drivers.CertMount, - ReadOnly: true, - } - - job.Spec.Template.Spec.Containers[0].VolumeMounts = append( - job.Spec.Template.Spec.Containers[0].VolumeMounts, - volumeMount, - ) - - volume := corev1.Volume{ - Name: utils.TLSCertMountVol, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: jobOption.CertSecretName, - }, - }, - } - - job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, volume) - - env := []corev1.EnvVar{ - { - Name: drivers.CertDirPath, - Value: drivers.CertMount, - }, - } - - job.Spec.Template.Spec.Containers[0].Env = env - } - - return job, nil -} - -// getVolumeDirectory gets the name of the directory on the host, under /var/lib/kubelet/pods//volumes/, -// where the specified volume lives. For volumes with a CSIVolumeSource, append "/mount" to the directory name. -func getVolumeDirectory(pvcName, pvcNamespace string) (string, error) { - fn := "getVolumeDirectory" - pvc, err := coreops.Instance().GetPersistentVolumeClaim(pvcName, pvcNamespace) - if err != nil { - errMsg := fmt.Sprintf("error fetching PVC %s/%s: %s", pvcNamespace, pvcName, err) - logrus.Errorf("%s: %v", fn, errMsg) - return "", fmt.Errorf(errMsg) - } - - pv, err := coreops.Instance().GetPersistentVolume(pvc.Spec.VolumeName) - if err != nil { - errMsg := fmt.Sprintf("error fetching PV %s/%s: %s", pvcNamespace, pvcName, err) - logrus.Errorf("%s: %v", fn, errMsg) - return "", fmt.Errorf(errMsg) - } - - // PV's been created with a CSI source. - if pv.Spec.CSI != nil { - return pvc.Spec.VolumeName + "/mount", nil - } - - // Check if the pv spec contains pv.kubernetes.io/migrated-to annotation, - // Then add the mount to the volume dir as it was provisioned by the CSI provisioner. - // From 1.23 k8s version onwards the in-tree storage provisioner driver are not supported. - // So all of them will migrated to use CSI provisioner. These driver will have above annotation and - // will not have the CSI section in the pv spec. - if pv.Annotations != nil { - annotations := pv.GetAnnotations() - if _, ok := annotations[k8shelper.AnnMigratedTo]; ok { - return pvc.Spec.VolumeName + "/mount", nil - } - } - - return pvc.Spec.VolumeName, nil -} diff --git a/vendor/github.com/portworx/kdmp/pkg/executor/common.go b/vendor/github.com/portworx/kdmp/pkg/executor/common.go index d79cc0705e..40b3d3371e 100644 --- a/vendor/github.com/portworx/kdmp/pkg/executor/common.go +++ b/vendor/github.com/portworx/kdmp/pkg/executor/common.go @@ -52,9 +52,9 @@ const ( // DefaultTimeout Max time a command will be retired before failing DefaultTimeout = 1 * time.Minute // BackupUID backup UID annotation - BackupUID = "portworx.io/backup-uid" - retrySleep = 10 * time.Second - maxRetry = 10 + BackupUID = "portworx.io/backup-uid" + retrySleep = 10 * time.Second + maxRetry = 10 lastKnownErrorLen = 1000 ) diff --git a/vendor/modules.txt b/vendor/modules.txt index 907a8052b4..18874c9431 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1021,8 +1021,8 @@ github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 ## explicit github.com/pmezard/go-difflib/difflib -# github.com/portworx/kdmp v0.4.1-0.20230830195819-704706dda5c7 => github.com/portworx/kdmp v0.4.1-0.20230830195819-704706dda5c7 -## explicit; go 1.19 +# github.com/portworx/kdmp v0.4.1-0.20230927104323-84564ac8b457 => github.com/portworx/kdmp v0.4.1-0.20230927104323-84564ac8b457 +## explicit; go 1.20 github.com/portworx/kdmp/pkg/apis/kdmp github.com/portworx/kdmp/pkg/apis/kdmp/v1alpha1 github.com/portworx/kdmp/pkg/client/clientset/versioned @@ -2401,7 +2401,7 @@ sigs.k8s.io/yaml # github.com/libopenstorage/autopilot-api => github.com/libopenstorage/autopilot-api v0.6.1-0.20210301232050-ca2633c6e114 # github.com/libopenstorage/openstorage => github.com/libopenstorage/openstorage v0.0.0-20230511212757-41751b27d69f # github.com/libopenstorage/secrets => github.com/libopenstorage/secrets v0.0.0-20220413195519-57d1c446c5e9 -# github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20230830195819-704706dda5c7 +# github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20230927104323-84564ac8b457 # github.com/portworx/sched-ops => github.com/portworx/sched-ops v1.20.4-rc1.0.20230922202555-69648f749f5a # github.com/portworx/torpedo => github.com/portworx/torpedo v0.0.0-20230927164041-818bdade92ff # gopkg.in/fsnotify.v1 v1.4.7 => github.com/fsnotify/fsnotify v1.4.7