cgroup: retry rmdir up to 5 seconds on EBUSY

on a busy system, the conmon process could take longer to complete or
to be reaped by the parent, leaving the cgroup busy.  If the rmdir
fails with EBUSY, try again up to 5 seconds before reporting an
error.

Closes: containers/podman#11946

Signed-off-by: Giuseppe Scrivano <[email protected]>

pkg/cgroups/cgroups.go

@@ -11,13 +11,15 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/containers/storage/pkg/unshare"
 	systemdDbus "github.com/coreos/go-systemd/v22/dbus"
 	"github.com/godbus/dbus/v5"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 )
 
 var (
@@ -527,7 +529,21 @@ func rmDirRecursively(path string) error {
 			}
 		}
 	}
-	if err := os.Remove(path); err != nil {
+
+	attempts := 0
+	for {
+		err := os.Remove(path)
+		if err == nil || os.IsNotExist(err) {
+			return nil
+		}
+		if errors.Is(err, unix.EBUSY) {
+			// attempt up to 5 seconds if the cgroup is busy
+			if attempts < 500 {
+				time.Sleep(time.Millisecond * 10)
+				attempts++
+				continue
+			}
+		}
 		if !os.IsNotExist(err) {
 			return errors.Wrapf(err, "remove %s", path)
 		}