cgroup: retry rmdir up to 5 seconds on EBUSY

on a busy system, the conmon process could take longer to complete or to be reaped by the parent, leaving the cgroup busy. If the rmdir fails with EBUSY, try again up to 5 seconds before reporting an error. Closes: containers/podman#11946 Signed-off-by: Giuseppe Scrivano <[email protected]>
giuseppe · Dec 14, 2021 · df9a7b3 · df9a7b3
1 parent bd6e86d
commit df9a7b3
Showing 1 changed file with 17 additions and 3 deletions.
diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go
@@ -11,13 +11,15 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/containers/storage/pkg/unshare"
 	systemdDbus "github.com/coreos/go-systemd/v22/dbus"
 	"github.com/godbus/dbus/v5"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 )
 
 var (
@@ -527,10 +529,22 @@ func rmDirRecursively(path string) error {
 			}
 		}
 	}
-	if err := os.Remove(path); err != nil {
-		if !os.IsNotExist(err) {
-			return errors.Wrapf(err, "remove %s", path)
+
+	attempts := 0
+	for {
+		err := os.Remove(path)
+		if err == nil || os.IsNotExist(err) {
+			return nil
+		}
+		if errors.Is(err, unix.EBUSY) {
+			// attempt up to 5 seconds if the cgroup is busy
+			if attempts < 500 {
+				time.Sleep(time.Millisecond * 10)
+				attempts++
+				continue
+			}
 		}
+		return errors.Wrapf(err, "remove %s", path)
 	}
 	return nil
 }