diff --git a/changelogs/unreleased/5145-jxun b/changelogs/unreleased/5145-jxun
new file mode 100644
index 0000000000..959ecf73df
--- /dev/null
+++ b/changelogs/unreleased/5145-jxun
@@ -0,0 +1 @@
+Delay CA file deletion in PVB controller.
\ No newline at end of file
diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go
index 322592f808..9cb6a3ae26 100644
--- a/pkg/controller/pod_volume_backup_controller.go
+++ b/pkg/controller/pod_volume_backup_controller.go
@@ -124,7 +124,11 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	if err != nil {
 		return r.updateStatusToFailed(ctx, &pvb, err, "building Restic command", log)
 	}
-	defer os.Remove(resticDetails.credsFile)
+
+	defer func() {
+		os.Remove(resticDetails.credsFile)
+		os.Remove(resticDetails.caCertFile)
+	}()
 
 	backupLocation := &velerov1api.BackupStorageLocation{}
 	if err := r.Client.Get(context.Background(), client.ObjectKey{
@@ -344,8 +348,6 @@ func (r *PodVolumeBackupReconciler) buildResticCommand(ctx context.Context, log
 		if err != nil {
 			log.WithError(err).Error("creating temporary caCert file")
 		}
-		defer os.Remove(details.caCertFile)
-
 	}
 	cmd.CACertFile = details.caCertFile