diff --git a/libcontainer/cgroups/fs2/fs2.go b/libcontainer/cgroups/fs2/fs2.go
index d845a1b5ebe..93a2aa59d1a 100644
--- a/libcontainer/cgroups/fs2/fs2.go
+++ b/libcontainer/cgroups/fs2/fs2.go
@@ -142,7 +142,10 @@ func (m *manager) Freeze(state configs.FreezerState) error {
 }
 
 func (m *manager) Destroy() error {
-	return os.RemoveAll(m.dirPath)
+	if err := os.Remove(m.dirPath); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
 }
 
 // GetPaths is for compatibility purpose and should be removed in future