From bb589bec24ede0ca7bc981a5a285fb66d7242655 Mon Sep 17 00:00:00 2001
From: Daniel J Walsh <dwalsh@redhat.com>
Date: Thu, 13 May 2021 08:45:29 -0400
Subject: [PATCH 01/13] Fix problem copying files when container is in host pid
 namespace

When attempting to copy files into and out of running containers
within the host pidnamespace, the code was attempting to join the
host pidns again, and getting an error. This was causing the podman
cp command to fail. Since we are already in the host pid namespace,
we should not be attempting to join.  This PR adds a check to see if
the container is in NOT host pid namespace, and only then attempts to
join.

Fixes: https://github.com/containers/podman/issues/9985

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>
---
 libpod/container_copy_linux.go | 25 +++++++++++++------
 libpod/container_inspect.go    | 23 ++++++++++++++++++
 test/e2e/cp_test.go            | 44 ++++++++++++++++++++++++++++++++++
 test/system/065-cp.bats        | 16 +++++++++++++
 4 files changed, 101 insertions(+), 7 deletions(-)

diff --git a/libpod/container_copy_linux.go b/libpod/container_copy_linux.go
index 5c275c641f..0ab3228295 100644
--- a/libpod/container_copy_linux.go
+++ b/libpod/container_copy_linux.go
@@ -237,21 +237,32 @@ func (c *Container) joinMountAndExec(ctx context.Context, f func() error) error
 		}
 		defer mountFD.Close()
 
-		pidFD, err := getFD(PIDNS)
+		inHostPidNS, err := c.inHostPidNS()
 		if err != nil {
-			errChan <- err
+			errChan <- errors.Wrap(err, "checking inHostPidNS")
 			return
 		}
-		defer pidFD.Close()
-		if err := unix.Unshare(unix.CLONE_NEWNS); err != nil {
-			errChan <- err
-			return
+		var pidFD *os.File
+		if !inHostPidNS {
+			pidFD, err = getFD(PIDNS)
+			if err != nil {
+				errChan <- err
+				return
+			}
+			defer pidFD.Close()
 		}
-		if err := unix.Setns(int(pidFD.Fd()), unix.CLONE_NEWPID); err != nil {
+
+		if err := unix.Unshare(unix.CLONE_NEWNS); err != nil {
 			errChan <- err
 			return
 		}
 
+		if pidFD != nil {
+			if err := unix.Setns(int(pidFD.Fd()), unix.CLONE_NEWPID); err != nil {
+				errChan <- err
+				return
+			}
+		}
 		if err := unix.Setns(int(mountFD.Fd()), unix.CLONE_NEWNS); err != nil {
 			errChan <- err
 			return
diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go
index 5b2103c92e..b38c356976 100644
--- a/libpod/container_inspect.go
+++ b/libpod/container_inspect.go
@@ -890,3 +890,26 @@ func (c *Container) generateInspectContainerHostConfig(ctrSpec *spec.Spec, named
 
 	return hostConfig, nil
 }
+
+// Return true if the container is running in the host's PID NS.
+func (c *Container) inHostPidNS() (bool, error) {
+	if c.config.PIDNsCtr != "" {
+		return false, nil
+	}
+	ctrSpec, err := c.specFromState()
+	if err != nil {
+		return false, err
+	}
+	if ctrSpec.Linux != nil {
+		// Locate the spec's PID namespace.
+		// If there is none, it's pid=host.
+		// If there is one and it has a path, it's "ns:".
+		// If there is no path, it's default - the empty string.
+		for _, ns := range ctrSpec.Linux.Namespaces {
+			if ns.Type == spec.PIDNamespace {
+				return false, nil
+			}
+		}
+	}
+	return true, nil
+}
diff --git a/test/e2e/cp_test.go b/test/e2e/cp_test.go
index c0fb3f8878..be49018334 100644
--- a/test/e2e/cp_test.go
+++ b/test/e2e/cp_test.go
@@ -91,6 +91,50 @@ var _ = Describe("Podman cp", func() {
 		Expect(roundtripContent).To(Equal(originalContent))
 	})
 
+	// Copy a file to the container, then back to the host in --pid=host
+	It("podman cp --pid=host file", func() {
+		SkipIfRootlessCgroupsV1("Not supported for rootless + CGroupsV1")
+		srcFile, err := ioutil.TempFile("", "")
+		Expect(err).To(BeNil())
+		defer srcFile.Close()
+		defer os.Remove(srcFile.Name())
+
+		originalContent := []byte("podman cp file test")
+		err = ioutil.WriteFile(srcFile.Name(), originalContent, 0644)
+		Expect(err).To(BeNil())
+
+		// Create a container. NOTE that container mustn't be running for copying.
+		session := podmanTest.Podman([]string{"create", "--pid=host", ALPINE, "top"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		name := session.OutputToString()
+
+		session = podmanTest.Podman([]string{"start", name})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		// The file will now be created (and written to).
+		session = podmanTest.Podman([]string{"cp", srcFile.Name(), name + ":foo"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		// Copy FROM the container.
+
+		destFile, err := ioutil.TempFile("", "")
+		Expect(err).To(BeNil())
+		defer destFile.Close()
+		defer os.Remove(destFile.Name())
+
+		session = podmanTest.Podman([]string{"cp", name + ":foo", destFile.Name()})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		// Now make sure the content matches.
+		roundtripContent, err := ioutil.ReadFile(destFile.Name())
+		Expect(err).To(BeNil())
+		Expect(roundtripContent).To(Equal(originalContent))
+	})
+
 	// Create a symlink in the container, use it as a copy destination and
 	// make sure that the link and the resolved path are accessible and
 	// give the right content.
diff --git a/test/system/065-cp.bats b/test/system/065-cp.bats
index 679cdc2095..24ac8118ec 100644
--- a/test/system/065-cp.bats
+++ b/test/system/065-cp.bats
@@ -130,6 +130,22 @@ load helpers
 }
 
 
+@test "podman cp file from/to host while --pid=host" {
+    if is_rootless && ! is_cgroupsv2; then
+        skip "'podman cp --pid=host' (rootless) only works with cgroups v2"
+    fi
+
+    srcdir=$PODMAN_TMPDIR/cp-pid-equals-host
+    mkdir -p $srcdir
+    touch $srcdir/hostfile
+
+    run_podman run --pid=host -d --name cpcontainer $IMAGE sleep infinity
+    run_podman cp $srcdir/hostfile cpcontainer:/tmp/hostfile
+    run_podman cp cpcontainer:/tmp/hostfile $srcdir/hostfile1
+    run_podman kill cpcontainer
+    run_podman rm -f cpcontainer
+}
+
 @test "podman cp file from container to host" {
     srcdir=$PODMAN_TMPDIR/cp-test-file-ctr-to-host
     mkdir -p $srcdir

From a9108ab258a2d70a80f336fbd5a42af8b284f4d2 Mon Sep 17 00:00:00 2001
From: Valentin Rothberg <rothberg@redhat.com>
Date: Wed, 19 May 2021 09:15:48 +0200
Subject: [PATCH 02/13] docs: generate systemd: XDG_RUNTIME_DIR

A conversation on the customer portal suggests that to add an extra note
about the requirement of XDG_RUNTIME_DIR to be set.

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>
---
 docs/source/markdown/podman-generate-systemd.1.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/markdown/podman-generate-systemd.1.md b/docs/source/markdown/podman-generate-systemd.1.md
index 4b77c57211..3565a42bcd 100644
--- a/docs/source/markdown/podman-generate-systemd.1.md
+++ b/docs/source/markdown/podman-generate-systemd.1.md
@@ -10,7 +10,7 @@ podman\-generate\-systemd - Generate systemd unit file(s) for a container or pod
 **podman generate systemd** will create a systemd unit file that can be used to control a container or pod.
 By default, the command will print the content of the unit files to stdout.
 
-_Note: If you use this command with the remote client, you would still have to place the generated units on the remote system._
+_Note: If you use this command with the remote client, you would still have to place the generated units on the remote system.  Moreover, please make sure that the XDG_RUNTIME_DIR environment variable is set.  If unset, you may set it via `export XDG_RUNTIME_DIR=/run/user/$(id -u)`._
 
 ## OPTIONS
 

From efa15b96c5d11796cdfbf2fdbe10664ec91b4c29 Mon Sep 17 00:00:00 2001
From: Valentin Rothberg <rothberg@redhat.com>
Date: Wed, 5 May 2021 15:43:50 +0200
Subject: [PATCH 03/13] add libimage events

libimage now supports events which `libpod.Runtime` now uses for image
events.

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

<MH: Removed vendor bits, kept other changes>

Signed-off-by: Matthew Heon <mheon@redhat.com>
---
 libpod/runtime.go           | 90 ++++++++++++++++++++++++++++++++-----
 test/system/090-events.bats | 36 +++++++++++++++
 2 files changed, 114 insertions(+), 12 deletions(-)

diff --git a/libpod/runtime.go b/libpod/runtime.go
index 80fe92b54f..2cf2b3f79b 100644
--- a/libpod/runtime.go
+++ b/libpod/runtime.go
@@ -13,6 +13,7 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/containers/common/libimage"
 	"github.com/containers/common/pkg/config"
@@ -67,17 +68,18 @@ type Runtime struct {
 	storageConfig storage.StoreOptions
 	storageSet    storageSet
 
-	state             State
-	store             storage.Store
-	storageService    *storageService
-	imageContext      *types.SystemContext
-	defaultOCIRuntime OCIRuntime
-	ociRuntimes       map[string]OCIRuntime
-	runtimeFlags      []string
-	netPlugin         ocicni.CNIPlugin
-	conmonPath        string
-	libimageRuntime   *libimage.Runtime
-	lockManager       lock.Manager
+	state                  State
+	store                  storage.Store
+	storageService         *storageService
+	imageContext           *types.SystemContext
+	defaultOCIRuntime      OCIRuntime
+	ociRuntimes            map[string]OCIRuntime
+	runtimeFlags           []string
+	netPlugin              ocicni.CNIPlugin
+	conmonPath             string
+	libimageRuntime        *libimage.Runtime
+	libimageEventsShutdown chan bool
+	lockManager            lock.Manager
 
 	// doRenumber indicates that the runtime should perform a lock renumber
 	// during initialization.
@@ -212,6 +214,8 @@ func newRuntimeFromConfig(ctx context.Context, conf *config.Config, options ...R
 		return nil, err
 	}
 
+	runtime.libimageEventsShutdown = make(chan bool)
+
 	return runtime, nil
 }
 
@@ -677,6 +681,62 @@ func (r *Runtime) GetConfig() (*config.Config, error) {
 	return config, nil
 }
 
+// libimageEventsMap translates a libimage event type to a libpod event status.
+var libimageEventsMap = map[libimage.EventType]events.Status{
+	libimage.EventTypeImagePull:    events.Pull,
+	libimage.EventTypeImagePush:    events.Push,
+	libimage.EventTypeImageRemove:  events.Remove,
+	libimage.EventTypeImageLoad:    events.LoadFromArchive,
+	libimage.EventTypeImageSave:    events.Save,
+	libimage.EventTypeImageTag:     events.Tag,
+	libimage.EventTypeImageUntag:   events.Untag,
+	libimage.EventTypeImageMount:   events.Mount,
+	libimage.EventTypeImageUnmount: events.Unmount,
+}
+
+// libimageEvents spawns a goroutine in the background which is listenting for
+// events on the libimage.Runtime.  The gourtine will be cleaned up implicitly
+// when the main() exists.
+func (r *Runtime) libimageEvents() {
+	toLibpodEventStatus := func(e *libimage.Event) events.Status {
+		status, found := libimageEventsMap[e.Type]
+		if !found {
+			return "Unknown"
+		}
+		return status
+	}
+
+	go func() {
+		eventChannel := r.libimageRuntime.EventChannel()
+
+		for {
+			// Make sure to read and write all events before
+			// checking if we're about to shutdown.
+			for len(eventChannel) > 0 {
+				libimageEvent := <-eventChannel
+				e := events.Event{
+					ID:     libimageEvent.ID,
+					Name:   libimageEvent.Name,
+					Status: toLibpodEventStatus(libimageEvent),
+					Time:   libimageEvent.Time,
+					Type:   events.Image,
+				}
+				if err := r.eventer.Write(e); err != nil {
+					logrus.Errorf("unable to write image event: %q", err)
+				}
+			}
+
+			select {
+			case <-r.libimageEventsShutdown:
+				return
+
+			default:
+				time.Sleep(100 * time.Millisecond)
+			}
+		}
+	}()
+}
+
 // DeferredShutdown shuts down the runtime without exposing any
 // errors. This is only meant to be used when the runtime is being
 // shutdown within a defer statement; else use Shutdown
@@ -716,7 +776,11 @@ func (r *Runtime) Shutdown(force bool) error {
 	// If no store was requested, it can be nil and there is no need to
 	// attempt to shut it down
 	if r.store != nil {
-		if _, err := r.store.Shutdown(force); err != nil {
+		// Wait for the events to be written.
+		r.libimageEventsShutdown <- true
+
+		// Note that the libimage runtime shuts down the store.
+		if err := r.libimageRuntime.Shutdown(force); err != nil {
 			lastError = errors.Wrapf(err, "error shutting down container storage")
 		}
 	}
@@ -842,6 +906,8 @@ func (r *Runtime) configureStore() error {
 		return err
 	}
 	r.libimageRuntime = libimageRuntime
+	// Run the libimage events routine.
+	r.libimageEvents()
 
 	return nil
 }
diff --git a/test/system/090-events.bats b/test/system/090-events.bats
index 8a9db41fac..19bee5506f 100644
--- a/test/system/090-events.bats
+++ b/test/system/090-events.bats
@@ -25,3 +25,39 @@ load helpers
     run_podman events --filter type=container --filter container=$cname --filter event=start --stream=false
     is "$output" "$expect" "filtering just by label"
 }
+
+@test "image events" {
+    skip_if_remote "FIXME: remove events on podman-remote seem to be broken"
+    pushedDir=$PODMAN_TMPDIR/dir
+    mkdir -p $pushedDir
+
+    tarball=$PODMAN_TMPDIR/ball.tar
+
+    run_podman image inspect --format "{{.ID}}" $IMAGE
+    imageID="$output"
+
+    t0=$(date --iso-8601=seconds)
+    tag=registry.com/$(random_string 10 | tr A-Z a-z)
+
+    # Force using the file backend since the journal backend is eating events
+    # (see containers/podman/pull/10219#issuecomment-842325032).
+    run_podman --events-backend=file push $IMAGE dir:$pushedDir
+    run_podman --events-backend=file save $IMAGE -o $tarball
+    run_podman --events-backend=file load -i $tarball
+    run_podman --events-backend=file pull docker-archive:$tarball
+    run_podman --events-backend=file tag $IMAGE $tag
+    run_podman --events-backend=file untag $IMAGE $tag
+    run_podman --events-backend=file tag $IMAGE $tag
+    run_podman --events-backend=file rmi $tag
+
+    run_podman --events-backend=file events --stream=false --filter type=image --since $t0
+    is "$output" ".*image push $imageID dir:$pushedDir
+.*image save $imageID $tarball
+.*image loadfromarchive *$tarball
+.*image pull *docker-archive:$tarball
+.*image tag $imageID $tag
+.*image untag $imageID $tag:latest
+.*image tag $imageID $tag
+.*image remove $imageID $tag.*" \
+       "podman events"
+}

From 18e917cdc361dc7f1badbce89b69e80a522062fe Mon Sep 17 00:00:00 2001
From: Jhon Honce <jhonce@redhat.com>
Date: Mon, 10 May 2021 10:22:33 -0700
Subject: [PATCH 04/13] Downgrade API service routing table logging

[NO TESTS NEEDED]

* Log the routing table output at Trace vs. Debug level. Reduce noise
  in debugging output.
* Tweak SDNotify message to report Warn when it fails. Previously
  failures were silent.

Signed-off-by: Jhon Honce <jhonce@redhat.com>
---
 pkg/api/server/server.go | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/pkg/api/server/server.go b/pkg/api/server/server.go
index e23448fd1e..972541bc6d 100644
--- a/pkg/api/server/server.go
+++ b/pkg/api/server/server.go
@@ -136,7 +136,7 @@ func newServer(runtime *libpod.Runtime, duration time.Duration, listener *net.Li
 		}
 	}
 
-	if logrus.IsLevelEnabled(logrus.DebugLevel) {
+	if logrus.IsLevelEnabled(logrus.TraceLevel) {
 		router.Walk(func(route *mux.Route, r *mux.Router, ancestors []*mux.Route) error { // nolint
 			path, err := route.GetPathTemplate()
 			if err != nil {
@@ -146,7 +146,7 @@ func newServer(runtime *libpod.Runtime, duration time.Duration, listener *net.Li
 			if err != nil {
 				methods = []string{"<N/A>"}
 			}
-			logrus.Debugf("Methods: %6s Path: %s", strings.Join(methods, ", "), path)
+			logrus.Tracef("Methods: %6s Path: %s", strings.Join(methods, ", "), path)
 			return nil
 		})
 	}
@@ -162,13 +162,12 @@ func setupSystemd() {
 	if len(os.Getenv("NOTIFY_SOCKET")) == 0 {
 		return
 	}
-	payload := fmt.Sprintf("MAINPID=%d", os.Getpid())
-	payload += "\n"
+	payload := fmt.Sprintf("MAINPID=%d\n", os.Getpid())
 	payload += daemon.SdNotifyReady
 	if sent, err := daemon.SdNotify(true, payload); err != nil {
 		logrus.Errorf("Error notifying systemd of Conmon PID: %s", err.Error())
-	} else if sent {
-		logrus.Debugf("Notify sent successfully")
+	} else if !sent {
+		logrus.Warn("SDNotify not sent successfully")
 	}
 
 	if err := os.Unsetenv("INVOCATION_ID"); err != nil {

From ac8b7ddd8dc2b3d4ac2da6b4ddd08b900eb02353 Mon Sep 17 00:00:00 2001
From: Daniel J Walsh <dwalsh@redhat.com>
Date: Fri, 21 May 2021 08:00:47 -0400
Subject: [PATCH 05/13] Drop container does not exist on removal to debugf

We have race conditions where a container can be removed
by two different processes when running podman --remove rm.

It can be cleaned up in the API or by the conmon executing
podman container cleanup.

When we fail to remove a container that does not exists we should
not be printing errors or warnings, we should just debug the fact.

[NO TESTS NEEDED] Since this is a race condition it is difficult to
test.

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>
---
 pkg/domain/infra/abi/containers.go    |  2 +-
 pkg/domain/infra/tunnel/containers.go | 13 ++++++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go
index d0a2b1bae3..237a434410 100644
--- a/pkg/domain/infra/abi/containers.go
+++ b/pkg/domain/infra/abi/containers.go
@@ -275,7 +275,7 @@ func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string,
 		case nil:
 			// remove container names that we successfully deleted
 			reports = append(reports, &report)
-		case define.ErrNoSuchCtr:
+		case define.ErrNoSuchCtr, define.ErrCtrExists:
 			// There is still a potential this is a libpod container
 			tmpNames = append(tmpNames, ctr)
 		default:
diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go
index 3830835cc5..74ced300a0 100644
--- a/pkg/domain/infra/tunnel/containers.go
+++ b/pkg/domain/infra/tunnel/containers.go
@@ -21,6 +21,7 @@ import (
 	"github.com/containers/podman/v3/pkg/errorhandling"
 	"github.com/containers/podman/v3/pkg/specgen"
 	"github.com/containers/podman/v3/pkg/util"
+	"github.com/containers/storage/types"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -580,7 +581,7 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri
 						if err := containers.Remove(ic.ClientCtx, ctr.ID, removeOptions); err != nil {
 							if errorhandling.Contains(err, define.ErrNoSuchCtr) ||
 								errorhandling.Contains(err, define.ErrCtrRemoved) {
-								logrus.Warnf("Container %s does not exist: %v", ctr.ID, err)
+								logrus.Debugf("Container %s does not exist: %v", ctr.ID, err)
 							} else {
 								logrus.Errorf("Error removing container %s: %v", ctr.ID, err)
 							}
@@ -613,8 +614,9 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri
 					rmOptions := new(containers.RemoveOptions).WithForce(false).WithVolumes(true)
 					if err := containers.Remove(ic.ClientCtx, ctr.ID, rmOptions); err != nil {
 						if errorhandling.Contains(err, define.ErrNoSuchCtr) ||
-							errorhandling.Contains(err, define.ErrCtrRemoved) {
-							logrus.Warnf("Container %s does not exist: %v", ctr.ID, err)
+							errorhandling.Contains(err, define.ErrCtrRemoved) ||
+							errorhandling.Contains(err, types.ErrLayerUnknown) {
+							logrus.Debugf("Container %s does not exist: %v", ctr.ID, err)
 						} else {
 							logrus.Errorf("Error removing container %s: %v", ctr.ID, err)
 						}
@@ -691,8 +693,9 @@ func (ic *ContainerEngine) ContainerRun(ctx context.Context, opts entities.Conta
 			if !shouldRestart {
 				if err := containers.Remove(ic.ClientCtx, con.ID, new(containers.RemoveOptions).WithForce(false).WithVolumes(true)); err != nil {
 					if errorhandling.Contains(err, define.ErrNoSuchCtr) ||
-						errorhandling.Contains(err, define.ErrCtrRemoved) {
-						logrus.Warnf("Container %s does not exist: %v", con.ID, err)
+						errorhandling.Contains(err, define.ErrCtrRemoved) ||
+						errorhandling.Contains(err, types.ErrLayerUnknown) {
+						logrus.Debugf("Container %s does not exist: %v", con.ID, err)
 					} else {
 						logrus.Errorf("Error removing container %s: %v", con.ID, err)
 					}

From c53638e9f61e61b1344cbf090c9766a1891c8a44 Mon Sep 17 00:00:00 2001
From: flouthoc <flouthoc.git@gmail.com>
Date: Wed, 19 May 2021 01:19:20 +0530
Subject: [PATCH 06/13] Podman info add support for status of cgroup
 controllers

Signed-off-by: flouthoc <flouthoc.git@gmail.com>
---
 docs/source/markdown/podman-info.1.md |  13 ++++
 libpod/define/info.go                 |  31 ++++----
 libpod/info.go                        |  44 ++++++-----
 pkg/cgroups/cgroups.go                | 106 ++++++++++++++++++++++++--
 test/e2e/info_test.go                 |  10 +++
 5 files changed, 163 insertions(+), 41 deletions(-)

diff --git a/docs/source/markdown/podman-info.1.md b/docs/source/markdown/podman-info.1.md
index 4af51d3ebe..227fbd92d6 100644
--- a/docs/source/markdown/podman-info.1.md
+++ b/docs/source/markdown/podman-info.1.md
@@ -32,6 +32,12 @@ $ podman info
 host:
   arch: amd64
   buildahVersion: 1.19.0-dev
+  cgroupControllers:
+  - cpuset
+  - cpu
+  - io
+  - memory
+  - pids
   cgroupManager: systemd
   cgroupVersion: v2
   conmon:
@@ -145,6 +151,13 @@ Run podman info with JSON formatted response:
     "buildahVersion": "1.19.0-dev",
     "cgroupManager": "systemd",
     "cgroupVersion": "v2",
+    "cgroupControllers": [
+      "cpuset",
+      "cpu",
+      "io",
+      "memory",
+      "pids"
+    ],
     "conmon": {
       "package": "conmon-2.0.22-2.fc33.x86_64",
       "path": "/usr/bin/conmon",
diff --git a/libpod/define/info.go b/libpod/define/info.go
index c9d6877c04..de709be74a 100644
--- a/libpod/define/info.go
+++ b/libpod/define/info.go
@@ -23,21 +23,22 @@ type SecurityInfo struct {
 
 // HostInfo describes the libpod host
 type HostInfo struct {
-	Arch           string           `json:"arch"`
-	BuildahVersion string           `json:"buildahVersion"`
-	CgroupManager  string           `json:"cgroupManager"`
-	CGroupsVersion string           `json:"cgroupVersion"`
-	Conmon         *ConmonInfo      `json:"conmon"`
-	CPUs           int              `json:"cpus"`
-	Distribution   DistributionInfo `json:"distribution"`
-	EventLogger    string           `json:"eventLogger"`
-	Hostname       string           `json:"hostname"`
-	IDMappings     IDMappings       `json:"idMappings,omitempty"`
-	Kernel         string           `json:"kernel"`
-	MemFree        int64            `json:"memFree"`
-	MemTotal       int64            `json:"memTotal"`
-	OCIRuntime     *OCIRuntimeInfo  `json:"ociRuntime"`
-	OS             string           `json:"os"`
+	Arch              string           `json:"arch"`
+	BuildahVersion    string           `json:"buildahVersion"`
+	CgroupManager     string           `json:"cgroupManager"`
+	CGroupsVersion    string           `json:"cgroupVersion"`
+	CgroupControllers []string         `json:"cgroupControllers"`
+	Conmon            *ConmonInfo      `json:"conmon"`
+	CPUs              int              `json:"cpus"`
+	Distribution      DistributionInfo `json:"distribution"`
+	EventLogger       string           `json:"eventLogger"`
+	Hostname          string           `json:"hostname"`
+	IDMappings        IDMappings       `json:"idMappings,omitempty"`
+	Kernel            string           `json:"kernel"`
+	MemFree           int64            `json:"memFree"`
+	MemTotal          int64            `json:"memTotal"`
+	OCIRuntime        *OCIRuntimeInfo  `json:"ociRuntime"`
+	OS                string           `json:"os"`
 	// RemoteSocket returns the UNIX domain socket the Podman service is listening on
 	RemoteSocket *RemoteSocket          `json:"remoteSocket,omitempty"`
 	RuntimeInfo  map[string]interface{} `json:"runtimeInfo,omitempty"`
diff --git a/libpod/info.go b/libpod/info.go
index 7a28a4cf79..461e39a48a 100644
--- a/libpod/info.go
+++ b/libpod/info.go
@@ -93,20 +93,33 @@ func (r *Runtime) hostInfo() (*define.HostInfo, error) {
 		return nil, errors.Wrapf(err, "error getting Seccomp profile path")
 	}
 
+	// CGroups version
+	unified, err := cgroups.IsCgroup2UnifiedMode()
+	if err != nil {
+		return nil, errors.Wrapf(err, "error reading cgroups mode")
+	}
+
+	// Get Map of all available controllers
+	availableControllers, err := cgroups.GetAvailableControllers(nil, unified)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error getting available cgroup controllers")
+	}
+
 	info := define.HostInfo{
-		Arch:           runtime.GOARCH,
-		BuildahVersion: buildah.Version,
-		CgroupManager:  r.config.Engine.CgroupManager,
-		Linkmode:       linkmode.Linkmode(),
-		CPUs:           runtime.NumCPU(),
-		Distribution:   hostDistributionInfo,
-		EventLogger:    r.eventer.String(),
-		Hostname:       host,
-		IDMappings:     define.IDMappings{},
-		Kernel:         kv,
-		MemFree:        mi.MemFree,
-		MemTotal:       mi.MemTotal,
-		OS:             runtime.GOOS,
+		Arch:              runtime.GOARCH,
+		BuildahVersion:    buildah.Version,
+		CgroupManager:     r.config.Engine.CgroupManager,
+		CgroupControllers: availableControllers,
+		Linkmode:          linkmode.Linkmode(),
+		CPUs:              runtime.NumCPU(),
+		Distribution:      hostDistributionInfo,
+		EventLogger:       r.eventer.String(),
+		Hostname:          host,
+		IDMappings:        define.IDMappings{},
+		Kernel:            kv,
+		MemFree:           mi.MemFree,
+		MemTotal:          mi.MemTotal,
+		OS:                runtime.GOOS,
 		Security: define.SecurityInfo{
 			AppArmorEnabled:     apparmor.IsEnabled(),
 			DefaultCapabilities: strings.Join(r.config.Containers.DefaultCapabilities, ","),
@@ -120,11 +133,6 @@ func (r *Runtime) hostInfo() (*define.HostInfo, error) {
 		SwapTotal:   mi.SwapTotal,
 	}
 
-	// CGroups version
-	unified, err := cgroups.IsCgroup2UnifiedMode()
-	if err != nil {
-		return nil, errors.Wrapf(err, "error reading cgroups mode")
-	}
 	cgroupVersion := "v1"
 	if unified {
 		cgroupVersion = "v2"
diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go
index aefb5183b4..911edeb5be 100644
--- a/pkg/cgroups/cgroups.go
+++ b/pkg/cgroups/cgroups.go
@@ -128,28 +128,118 @@ func init() {
 // getAvailableControllers get the available controllers
 func getAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]controller, error) {
 	if cgroup2 {
-		return nil, fmt.Errorf("getAvailableControllers not implemented yet for cgroup v2")
+		controllers := []controller{}
+		subtreeControl := cgroupRoot + "/cgroup.subtree_control"
+		// rootless cgroupv2: check available controllers for current user ,systemd or servicescope will inherit
+		if rootless.IsRootless() {
+			userSlice, err := getCgroupPathForCurrentProcess()
+			if err != nil {
+				return controllers, err
+			}
+			//userSlice already contains '/' so not adding here
+			basePath := cgroupRoot + userSlice
+			subtreeControl = fmt.Sprintf("%s/cgroup.subtree_control", basePath)
+		}
+		subtreeControlBytes, err := ioutil.ReadFile(subtreeControl)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed while reading controllers for cgroup v2 from %q", subtreeControl)
+		}
+		for _, controllerName := range strings.Fields(string(subtreeControlBytes)) {
+			c := controller{
+				name:    controllerName,
+				symlink: false,
+			}
+			controllers = append(controllers, c)
+		}
+		return controllers, nil
 	}
 
-	infos, err := ioutil.ReadDir(cgroupRoot)
-	if err != nil {
-		return nil, err
-	}
+	subsystems, _ := cgroupV1GetAllSubsystems()
 	controllers := []controller{}
-	for _, i := range infos {
-		name := i.Name()
+	// cgroupv1 and rootless: No subsystem is available: delegation is unsafe.
+	if rootless.IsRootless() {
+		return controllers, nil
+	}
+
+	for _, name := range subsystems {
 		if _, found := exclude[name]; found {
 			continue
 		}
+		isSymLink := false
+		fileInfo, err := os.Stat(cgroupRoot + "/" + name)
+		if err != nil {
+			isSymLink = !fileInfo.IsDir()
+		}
 		c := controller{
 			name:    name,
-			symlink: !i.IsDir(),
+			symlink: isSymLink,
 		}
 		controllers = append(controllers, c)
 	}
+
 	return controllers, nil
 }
 
+// GetAvailableControllers get string:bool map of all the available controllers
+func GetAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]string, error) {
+	availableControllers, err := getAvailableControllers(exclude, cgroup2)
+	if err != nil {
+		return nil, err
+	}
+	controllerList := []string{}
+	for _, controller := range availableControllers {
+		controllerList = append(controllerList, controller.name)
+	}
+
+	return controllerList, nil
+}
+
+func cgroupV1GetAllSubsystems() ([]string, error) {
+	f, err := os.Open("/proc/cgroups")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	subsystems := []string{}
+
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		text := s.Text()
+		if text[0] != '#' {
+			parts := strings.Fields(text)
+			if len(parts) >= 4 && parts[3] != "0" {
+				subsystems = append(subsystems, parts[0])
+			}
+		}
+	}
+	if err := s.Err(); err != nil {
+		return nil, err
+	}
+	return subsystems, nil
+}
+
+func getCgroupPathForCurrentProcess() (string, error) {
+	path := fmt.Sprintf("/proc/%d/cgroup", os.Getpid())
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	cgroupPath := ""
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		text := s.Text()
+		procEntries := strings.SplitN(text, "::", 2)
+		cgroupPath = procEntries[1]
+	}
+	if err := s.Err(); err != nil {
+		return cgroupPath, err
+	}
+	return cgroupPath, nil
+}
+
 // getCgroupv1Path is a helper function to get the cgroup v1 path
 func (c *CgroupControl) getCgroupv1Path(name string) string {
 	return filepath.Join(cgroupRoot, name, c.path)
diff --git a/test/e2e/info_test.go b/test/e2e/info_test.go
index 60136bcc24..f5b70d6bf4 100644
--- a/test/e2e/info_test.go
+++ b/test/e2e/info_test.go
@@ -135,4 +135,14 @@ var _ = Describe("Podman Info", func() {
 			Expect(session.OutputToString()).To(ContainSubstring("false"))
 		}
 	})
+
+	It("Podman info must contain cgroupControllers with ReleventControllers", func() {
+		SkipIfRootless("Hard to tell which controllers are going to be enabled for rootless")
+		SkipIfRootlessCgroupsV1("Disable cgroups not supported on cgroupv1 for rootless users")
+		session := podmanTest.Podman([]string{"info", "--format", "{{.Host.CgroupControllers}}"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+		Expect(session.OutputToString()).To(ContainSubstring("memory"))
+		Expect(session.OutputToString()).To(ContainSubstring("pids"))
+	})
 })

From ba884865c58d83d7a020bfb7fe90a182ebbe1405 Mon Sep 17 00:00:00 2001
From: Daniel J Walsh <dwalsh@redhat.com>
Date: Thu, 20 May 2021 16:09:40 -0400
Subject: [PATCH 07/13] Handle hard links in remote builds

Fixes: https://github.com/containers/podman/issues/9893

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>
---
 Makefile                             |  3 ++
 pkg/bindings/images/build.go         | 44 ++++++++++++++++++++--------
 pkg/bindings/images/build_unix.go    | 16 ++++++++++
 pkg/bindings/images/build_windows.go |  9 ++++++
 test/system/070-build.bats           | 20 +++++++++++++
 5 files changed, 80 insertions(+), 12 deletions(-)
 create mode 100644 pkg/bindings/images/build_unix.go
 create mode 100644 pkg/bindings/images/build_windows.go

diff --git a/Makefile b/Makefile
index 25644dffd9..a0b12e7de6 100644
--- a/Makefile
+++ b/Makefile
@@ -382,6 +382,9 @@ bin/podman.cross.%: .gopathok
 .PHONY: local-cross
 local-cross: $(CROSS_BUILD_TARGETS) ## Cross compile podman binary for multiple architectures
 
+.PHONY: cross
+cross: local-cross
+
 # Update nix/nixpkgs.json its latest stable commit
 .PHONY: nixpkgs
 nixpkgs:
diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go
index f5e7c0c98e..b56afbceb5 100644
--- a/pkg/bindings/images/build.go
+++ b/pkg/bindings/images/build.go
@@ -28,6 +28,11 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+type devino struct {
+	Dev uint64
+	Ino uint64
+}
+
 var (
 	iidRegex = regexp.MustCompile(`^[0-9a-f]{12}`)
 )
@@ -402,7 +407,7 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) {
 		defer pw.Close()
 		defer gw.Close()
 		defer tw.Close()
-
+		seen := make(map[devino]string)
 		for _, src := range sources {
 			s, err := filepath.Abs(src)
 			if err != nil {
@@ -431,25 +436,40 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) {
 				}
 
 				if info.Mode().IsRegular() { // add file item
-					f, lerr := os.Open(path)
-					if lerr != nil {
-						return lerr
+					di, isHardLink := checkHardLink(info)
+					if err != nil {
+						return err
 					}
 
-					hdr, lerr := tar.FileInfoHeader(info, name)
-					if lerr != nil {
-						f.Close()
-						return lerr
+					hdr, err := tar.FileInfoHeader(info, "")
+					if err != nil {
+						return err
+					}
+					orig, ok := seen[di]
+					if ok {
+						hdr.Typeflag = tar.TypeLink
+						hdr.Linkname = orig
+						hdr.Size = 0
+
+						return tw.WriteHeader(hdr)
+					}
+					f, err := os.Open(path)
+					if err != nil {
+						return err
 					}
+
 					hdr.Name = name
-					if lerr := tw.WriteHeader(hdr); lerr != nil {
+					if err := tw.WriteHeader(hdr); err != nil {
 						f.Close()
-						return lerr
+						return err
 					}
 
-					_, cerr := io.Copy(tw, f)
+					_, err = io.Copy(tw, f)
 					f.Close()
-					return cerr
+					if err == nil && isHardLink {
+						seen[di] = name
+					}
+					return err
 				} else if info.Mode().IsDir() { // add folders
 					hdr, lerr := tar.FileInfoHeader(info, name)
 					if lerr != nil {
diff --git a/pkg/bindings/images/build_unix.go b/pkg/bindings/images/build_unix.go
new file mode 100644
index 0000000000..0afb1deb6f
--- /dev/null
+++ b/pkg/bindings/images/build_unix.go
@@ -0,0 +1,16 @@
+// +build !windows
+
+package images
+
+import (
+	"os"
+	"syscall"
+)
+
+func checkHardLink(fi os.FileInfo) (devino, bool) {
+	st := fi.Sys().(*syscall.Stat_t)
+	return devino{
+		Dev: uint64(st.Dev),
+		Ino: uint64(st.Ino),
+	}, st.Nlink > 1
+}
diff --git a/pkg/bindings/images/build_windows.go b/pkg/bindings/images/build_windows.go
new file mode 100644
index 0000000000..bd71d1bf0d
--- /dev/null
+++ b/pkg/bindings/images/build_windows.go
@@ -0,0 +1,9 @@
+package images
+
+import (
+	"os"
+)
+
+func checkHardLink(fi os.FileInfo) (devino, bool) {
+	return devino{}, false
+}
diff --git a/test/system/070-build.bats b/test/system/070-build.bats
index d2d56c051a..9e15590139 100644
--- a/test/system/070-build.bats
+++ b/test/system/070-build.bats
@@ -766,6 +766,26 @@ EOF
     is "$output" ".*/tmp/bogus: no such file or directory"
 }
 
+@test "podman build COPY hardlinks " {
+    tmpdir=$PODMAN_TMPDIR/build-test
+    mkdir -p $tmpdir
+
+    dockerfile=$tmpdir/Dockerfile
+    cat >$dockerfile <<EOF
+FROM $IMAGE
+COPY . /test
+EOF
+    ln $dockerfile $tmpdir/hardlink
+
+    run_podman build -t build_test $tmpdir
+    run_podman run --rm build_test stat -c '%i' /test/Dockerfile
+    dinode=$output
+    run_podman run --rm build_test stat -c '%i' /test/hardlink
+    is "$output"   "$dinode"   "COPY hardlinks work"
+
+    run_podman rmi -f build_test
+}
+
 function teardown() {
     # A timeout or other error in 'build' can leave behind stale images
     # that podman can't even see and which will cascade into subsequent

From 5eecc2761b4439d8b8f41da38e7669415354d63f Mon Sep 17 00:00:00 2001
From: Daniel J Walsh <dwalsh@redhat.com>
Date: Fri, 21 May 2021 16:02:13 -0400
Subject: [PATCH 08/13] Improve OCI Runtime error

ErrOCIRuntimeNotFound error is misleading. Try to make it more
understandable to the user that the OCI Runtime IE crun or runc is not
missing, but the command they attempted to run within the container is
missing.

[NO TESTS NEEDED] Regular tests should handle this.

Fixes: https://github.com/containers/podman/issues/10432

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>
---
 libpod/define/errors.go  | 2 +-
 test/system/030-run.bats | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libpod/define/errors.go b/libpod/define/errors.go
index 81bf5f69c9..9fd210eed3 100644
--- a/libpod/define/errors.go
+++ b/libpod/define/errors.go
@@ -152,7 +152,7 @@ var (
 
 	// ErrOCIRuntimeNotFound indicates the OCI runtime attempted to invoke a command
 	// that was not found
-	ErrOCIRuntimeNotFound = errors.New("OCI not found")
+	ErrOCIRuntimeNotFound = errors.New("OCI runtime attempted to invoke a command that was not found")
 
 	// ErrOCIRuntimeUnavailable indicates that the OCI runtime associated to a container
 	// could not be found in the configuration
diff --git a/test/system/030-run.bats b/test/system/030-run.bats
index 9a136ff139..ccae703e56 100644
--- a/test/system/030-run.bats
+++ b/test/system/030-run.bats
@@ -14,7 +14,7 @@ load helpers
     # ...but check the configured runtime engine, and switch to crun as needed
     run_podman info --format '{{ .Host.OCIRuntime.Path }}'
     if expr "$output" : ".*/crun"; then
-        err_no_such_cmd="Error: executable file.* not found in \$PATH: No such file or directory: OCI not found"
+        err_no_such_cmd="Error: executable file.* not found in \$PATH: No such file or directory: OCI runtime attempted to invoke a command that was not found"
         err_no_exec_dir="Error: open executable: Operation not permitted: OCI permission denied"
     fi
 

From 934f36df5e94dd976e73f4a5cd99f092ecba052a Mon Sep 17 00:00:00 2001
From: Daniel J Walsh <dwalsh@redhat.com>
Date: Mon, 24 May 2021 15:41:51 -0400
Subject: [PATCH 09/13] Handle hard links in different directories

Fixes: https://github.com/containers/podman/issues/10444

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>
---
 pkg/bindings/images/build.go |  2 +-
 test/system/070-build.bats   | 14 +++++++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go
index b56afbceb5..346d55c47f 100644
--- a/pkg/bindings/images/build.go
+++ b/pkg/bindings/images/build.go
@@ -450,7 +450,7 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) {
 						hdr.Typeflag = tar.TypeLink
 						hdr.Linkname = orig
 						hdr.Size = 0
-
+						hdr.Name = name
 						return tw.WriteHeader(hdr)
 					}
 					f, err := os.Open(path)
diff --git a/test/system/070-build.bats b/test/system/070-build.bats
index 9e15590139..0f3f3fa7fa 100644
--- a/test/system/070-build.bats
+++ b/test/system/070-build.bats
@@ -768,19 +768,27 @@ EOF
 
 @test "podman build COPY hardlinks " {
     tmpdir=$PODMAN_TMPDIR/build-test
-    mkdir -p $tmpdir
+    subdir=$tmpdir/subdir
+    subsubdir=$subdir/subsubdir
+    mkdir -p $subsubdir
 
     dockerfile=$tmpdir/Dockerfile
     cat >$dockerfile <<EOF
 FROM $IMAGE
 COPY . /test
 EOF
-    ln $dockerfile $tmpdir/hardlink
+    ln $dockerfile $tmpdir/hardlink1
+    ln $dockerfile $subdir/hardlink2
+    ln $dockerfile $subsubdir/hardlink3
 
     run_podman build -t build_test $tmpdir
     run_podman run --rm build_test stat -c '%i' /test/Dockerfile
     dinode=$output
-    run_podman run --rm build_test stat -c '%i' /test/hardlink
+    run_podman run --rm build_test stat -c '%i' /test/hardlink1
+    is "$output"   "$dinode"   "COPY hardlinks work"
+    run_podman run --rm build_test stat -c '%i' /test/subdir/hardlink2
+    is "$output"   "$dinode"   "COPY hardlinks work"
+    run_podman run --rm build_test stat -c '%i' /test/subdir/subsubdir/hardlink3
     is "$output"   "$dinode"   "COPY hardlinks work"
 
     run_podman rmi -f build_test

From 69bae4774bc8bb25943f3eeb12684d6e581809fe Mon Sep 17 00:00:00 2001
From: Valentin Rothberg <rothberg@redhat.com>
Date: Tue, 25 May 2021 14:00:06 +0200
Subject: [PATCH 10/13] docs: --cert-dir: point to containers-certs.d(5)

Point to containers-certs.d(5) for details on the default paths, the
lookup logic and the structure of these directories.  Previously, the
man pages stated that the default path would be in `/etc/containers/...`
which is not entirely and a red herring for users (see #10116).

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>
---
 docs/source/markdown/podman-build.1.md              | 4 ++--
 docs/source/markdown/podman-container-runlabel.1.md | 4 ++--
 docs/source/markdown/podman-image-sign.1.md         | 4 ++--
 docs/source/markdown/podman-login.1.md              | 4 ++--
 docs/source/markdown/podman-manifest-add.1.md       | 4 ++--
 docs/source/markdown/podman-manifest-push.1.md      | 4 ++--
 docs/source/markdown/podman-play-kube.1.md          | 4 ++--
 docs/source/markdown/podman-pull.1.md               | 4 ++--
 docs/source/markdown/podman-push.1.md               | 4 ++--
 9 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/docs/source/markdown/podman-build.1.md b/docs/source/markdown/podman-build.1.md
index f173b24b4f..c65fbccb9f 100644
--- a/docs/source/markdown/podman-build.1.md
+++ b/docs/source/markdown/podman-build.1.md
@@ -111,7 +111,7 @@ given.
 #### **--cert-dir**=*path*
 
 Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
-Default certificates directory is _/etc/containers/certs.d_. (This option is not available with the remote Podman client)
+Please refer to containers-certs.d(5) for details. (This option is not available with the remote Podman client)
 
 #### **--cgroup-parent**=*path*
 
@@ -972,7 +972,7 @@ If you are using `useradd` within your build script, you should pass the
 useradd to stop creating the lastlog file.
 
 ## SEE ALSO
-podman(1), buildah(1), containers-registries.conf(5), crun(8), runc(8), useradd(8), podman-ps(1), podman-rm(1)
+podman(1), buildah(1), containers-certs.d(5), containers-registries.conf(5), crun(8), runc(8), useradd(8), podman-ps(1), podman-rm(1)
 
 ## HISTORY
 Aug 2020, Additional options and .dockerignore added by Dan Walsh `<dwalsh@redhat.com>`
diff --git a/docs/source/markdown/podman-container-runlabel.1.md b/docs/source/markdown/podman-container-runlabel.1.md
index aee849cbdc..e343a12fe4 100644
--- a/docs/source/markdown/podman-container-runlabel.1.md
+++ b/docs/source/markdown/podman-container-runlabel.1.md
@@ -57,7 +57,7 @@ The runlabel command will not execute if --display is specified.
 #### **--cert-dir**=*path*
 
 Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
-Default certificates directory is _/etc/containers/certs.d_. (This option is not available with the remote Podman client)
+Please refer to containers-certs.d(5) for details. (This option is not available with the remote Podman client)
 
 #### **--creds**=*[username[:password]]*
 
@@ -105,7 +105,7 @@ $ sudo podman container runlabel --display run foobar
 ```
 
 ## SEE ALSO
-podman(1)
+podman(1), containers-certs.d(5)
 
 ## HISTORY
 September 2018, Originally compiled by Brent Baude (bbaude at redhat dot com)
diff --git a/docs/source/markdown/podman-image-sign.1.md b/docs/source/markdown/podman-image-sign.1.md
index b9addc062f..616aab8ff4 100644
--- a/docs/source/markdown/podman-image-sign.1.md
+++ b/docs/source/markdown/podman-image-sign.1.md
@@ -26,7 +26,7 @@ Sign all the manifests of the multi-architecture image (default false).
 #### **--cert-dir**=*path*
 
 Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
-Default certificates directory is _/etc/containers/certs.d_. (This option is not available with the remote Podman client)
+Please refer to containers-certs.d(5) for details. (This option is not available with the remote Podman client)
 
 #### **--directory**, **-d**=*dir*
 
@@ -61,7 +61,7 @@ the signature will be written into sub-directories of
 the signature will be 'read' from that same location on a pull-related function.
 
 ## SEE ALSO
-containers-registries.d(5)
+containers-certs.d(5), containers-registries.d(5)
 
 ## HISTORY
 November 2018, Originally compiled by Qi Wang (qiwan at redhat dot com)
diff --git a/docs/source/markdown/podman-login.1.md b/docs/source/markdown/podman-login.1.md
index 2748690420..88d025c706 100644
--- a/docs/source/markdown/podman-login.1.md
+++ b/docs/source/markdown/podman-login.1.md
@@ -54,7 +54,7 @@ Return the logged-in user for the registry.  Return error if no login is found.
 #### **--cert-dir**=*path*
 
 Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
-Default certificates directory is _/etc/containers/certs.d_. (This option is not available with the remote Podman client)
+Please refer to containers-certs.d(5) for details. (This option is not available with the remote Podman client)
 
 #### **--tls-verify**=*true|false*
 
@@ -108,7 +108,7 @@ Login Succeeded!
 ```
 
 ## SEE ALSO
-podman(1), podman-logout(1), containers-auth.json(5), containers-registries.conf(5)
+podman(1), podman-logout(1), containers-auth.json(5), containers-certs.d(5), containers-registries.conf(5)
 
 ## HISTORY
 August 2017, Originally compiled by Urvashi Mohnani <umohnani@redhat.com>
diff --git a/docs/source/markdown/podman-manifest-add.1.md b/docs/source/markdown/podman-manifest-add.1.md
index 59d5e5d2c2..3a0e34ee51 100644
--- a/docs/source/markdown/podman-manifest-add.1.md
+++ b/docs/source/markdown/podman-manifest-add.1.md
@@ -44,7 +44,7 @@ environment variable. `export REGISTRY_AUTH_FILE=path`
 #### **--cert-dir**=*path*
 
 Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
-Default certificates directory is _/etc/containers/certs.d_. (This option is not available with the remote Podman client)
+Please refer to containers-certs.d(5) for details. (This option is not available with the remote Podman client)
 
 #### **--creds**=*creds*
 
@@ -110,4 +110,4 @@ podman manifest add --arch arm64 --variant v8 mylist:v1.11 docker://71c201d10fff
 ```
 
 ## SEE ALSO
-podman(1), podman-manifest(1), podman-manifest-create(1), podman-manifest-inspect(1), podman-manifest-push(1), podman-manifest-remove(1), podman-rmi(1)
+podman(1), podman-manifest(1), podman-manifest-create(1), podman-manifest-inspect(1), podman-manifest-push(1), podman-manifest-remove(1), podman-rmi(1), containers-certs.d(5)
diff --git a/docs/source/markdown/podman-manifest-push.1.md b/docs/source/markdown/podman-manifest-push.1.md
index 5097ada64e..2cf1cc375f 100644
--- a/docs/source/markdown/podman-manifest-push.1.md
+++ b/docs/source/markdown/podman-manifest-push.1.md
@@ -30,7 +30,7 @@ environment variable. `export REGISTRY_AUTH_FILE=path`
 #### **--cert-dir**=*path*
 
 Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
-Default certificates directory is _/etc/containers/certs.d_. (This option is not available with the remote Podman client)
+Please refer to containers-certs.d(5) for details. (This option is not available with the remote Podman client)
 
 #### **--creds**=*creds*
 
@@ -107,4 +107,4 @@ podman manifest push mylist:v1.11 docker://registry.example.org/mylist:v1.11
 ```
 
 ## SEE ALSO
-podman(1), podman-manifest(1), podman-manifest-add(1), podman-manifest-create(1), podman-manifest-inspect(1), podman-manifest-remove(1), podman-rmi(1)
+podman(1), podman-manifest(1), podman-manifest-add(1), podman-manifest-create(1), podman-manifest-inspect(1), podman-manifest-remove(1), podman-rmi(1), containers-certs.d(5)
diff --git a/docs/source/markdown/podman-play-kube.1.md b/docs/source/markdown/podman-play-kube.1.md
index 98b303ae75..ad5ae7e4cc 100644
--- a/docs/source/markdown/podman-play-kube.1.md
+++ b/docs/source/markdown/podman-play-kube.1.md
@@ -48,7 +48,7 @@ environment variable. `export REGISTRY_AUTH_FILE=path`
 #### **--cert-dir**=*path*
 
 Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
-Default certificates directory is _/etc/containers/certs.d_. (This option is not available with the remote Podman client)
+Please refer to containers-certs.d(5) for details. (This option is not available with the remote Podman client)
 
 #### **--configmap**=*path*
 
@@ -132,7 +132,7 @@ $ podman play kube demo.yml --network cni1,cni2
 Please take into account that CNI networks must be created first using podman-network-create(1).
 
 ## SEE ALSO
-podman(1), podman-container(1), podman-pod(1), podman-generate-kube(1), podman-play(1), podman-network-create(1)
+podman(1), podman-container(1), podman-pod(1), podman-generate-kube(1), podman-play(1), podman-network-create(1), containers-certs.d(5)
 
 ## HISTORY
 December 2018, Originally compiled by Brent Baude (bbaude at redhat dot com)
diff --git a/docs/source/markdown/podman-pull.1.md b/docs/source/markdown/podman-pull.1.md
index 9b42d07ac1..54f990601f 100644
--- a/docs/source/markdown/podman-pull.1.md
+++ b/docs/source/markdown/podman-pull.1.md
@@ -80,7 +80,7 @@ environment variable. `export REGISTRY_AUTH_FILE=path`
 #### **--cert-dir**=*path*
 
 Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
-Default certificates directory is _/etc/containers/certs.d_. (This option is not available with the remote Podman client)
+Please refer to containers-certs.d(5) for details. (This option is not available with the remote Podman client)
 
 #### **--creds**=*[username[:password]]*
 
@@ -217,7 +217,7 @@ registries.conf is the configuration file which specifies which container regist
 NOTE: Use the environment variable `TMPDIR` to change the temporary storage location of downloaded container images. Podman defaults to use `/var/tmp`.
 
 ## SEE ALSO
-podman(1), podman-push(1), podman-login(1), containers-registries.conf(5)
+podman(1), podman-push(1), podman-login(1), containers-certs.d(5), containers-registries.conf(5)
 
 ## HISTORY
 July 2017, Originally compiled by Urvashi Mohnani <umohnani@redhat.com>
diff --git a/docs/source/markdown/podman-push.1.md b/docs/source/markdown/podman-push.1.md
index f42ee10203..e26d738916 100644
--- a/docs/source/markdown/podman-push.1.md
+++ b/docs/source/markdown/podman-push.1.md
@@ -71,7 +71,7 @@ value can be entered.  The password is entered without echo.
 #### **--cert-dir**=*path*
 
 Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry.
-Default certificates directory is _/etc/containers/certs.d_. (This option is not available with the remote Podman client)
+Please refer to containers-certs.d(5) for details. (This option is not available with the remote Podman client)
 
 #### **--compress**
 
@@ -161,4 +161,4 @@ Storing signatures
 ```
 
 ## SEE ALSO
-podman(1), podman-pull(1), podman-login(1)
+podman(1), podman-pull(1), podman-login(1), containers-certs.d(5)

From 6c9de93823344fc33f6bf1be1039ce7654a5422f Mon Sep 17 00:00:00 2001
From: Daniel J Walsh <dwalsh@redhat.com>
Date: Tue, 25 May 2021 06:15:32 -0400
Subject: [PATCH 11/13] Fix race condition in running ls container in a pod

All of the tests has an assumption that RunLsContainer and RunLsContainerInPod completes
the container before returning.  But since the container is running
in back ground mode, the container could be still running before tools
attempt to remove it. Removing the "-d" from the command fixes the
container to match the assumption.

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>
---
 test/e2e/common_test.go | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go
index 3593450965..7ffee961ce 100644
--- a/test/e2e/common_test.go
+++ b/test/e2e/common_test.go
@@ -408,7 +408,14 @@ func (p *PodmanTestIntegration) RunLsContainer(name string) (*PodmanSessionInteg
 	podmanArgs = append(podmanArgs, "-d", ALPINE, "ls")
 	session := p.Podman(podmanArgs)
 	session.WaitWithDefaultTimeout()
-	return session, session.ExitCode(), session.OutputToString()
+	if session.ExitCode() != 0 {
+		return session, session.ExitCode(), session.OutputToString()
+	}
+	cid := session.OutputToString()
+
+	wsession := p.Podman([]string{"wait", cid})
+	wsession.WaitWithDefaultTimeout()
+	return session, wsession.ExitCode(), cid
 }
 
 // RunNginxWithHealthCheck runs the alpine nginx container with an optional name and adds a healthcheck into it
@@ -431,7 +438,14 @@ func (p *PodmanTestIntegration) RunLsContainerInPod(name, pod string) (*PodmanSe
 	podmanArgs = append(podmanArgs, "-d", ALPINE, "ls")
 	session := p.Podman(podmanArgs)
 	session.WaitWithDefaultTimeout()
-	return session, session.ExitCode(), session.OutputToString()
+	if session.ExitCode() != 0 {
+		return session, session.ExitCode(), session.OutputToString()
+	}
+	cid := session.OutputToString()
+
+	wsession := p.Podman([]string{"wait", cid})
+	wsession.WaitWithDefaultTimeout()
+	return session, wsession.ExitCode(), cid
 }
 
 // BuildImage uses podman build and buildah to build an image

From ee5dd06035eff9765c739228fb6b9a2cc0a2bf5b Mon Sep 17 00:00:00 2001
From: Daniel J Walsh <dwalsh@redhat.com>
Date: Tue, 25 May 2021 10:43:49 -0400
Subject: [PATCH 12/13] Fix race on podman start --all

Make sure all containers exit after start

There is a race condition in that container could still be running when
we attempt to remove them.

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>
---
 test/system/045-start.bats | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/system/045-start.bats b/test/system/045-start.bats
index 542f9d1c27..3e0118dba8 100644
--- a/test/system/045-start.bats
+++ b/test/system/045-start.bats
@@ -25,6 +25,8 @@ load helpers
         die "podman start --all restarted a running container"
     fi
 
+    run_podman wait $cid_none_implicit $cid_none_explicit $cid_on_failure
+
     run_podman rm $cid_none_implicit $cid_none_explicit $cid_on_failure
     run_podman stop -t 1 $cid_always
     run_podman rm $cid_always

From f49023031d85ddf5700855ac374e63e2bbaa653e Mon Sep 17 00:00:00 2001
From: Matthew Heon <mheon@redhat.com>
Date: Tue, 25 May 2021 15:28:10 -0400
Subject: [PATCH 13/13] Update release notes for v3.2.0-RC3

Signed-off-by: Matthew Heon <mheon@redhat.com>
---
 RELEASE_NOTES.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 7f833ac2de..a08439b920 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -33,7 +33,7 @@
 - The `podman play kube` now treats environment variables configured as references to a `ConfigMap` as mandatory unless the `optional` parameter was set; this better matches the behavior of Kubernetes.
 - Podman now supports the `--context=default` flag from Docker as a no-op for compatibility purposes.
 - When Podman is run as root, but without `CAP_SYS_ADMIN` being available, it will run in a user namespace using the same code as rootless Podman (instead of failing outright).
-- The `podman info` command now includes the path of the Seccomp profile Podman is using, and whether Podman is connected to a remote service or running containers locally.
+- The `podman info` command now includes the path of the Seccomp profile Podman is using, available cgroup controllers, and whether Podman is connected to a remote service or running containers locally.
 - Containers created with the `--rm` option now automatically use the `volatile` storage flag when available for their root filesystems, causing them not to write changes to disk as often as they will be removed at completion anyways. This should result in improved performance.
 - The `podman generate systemd --new` command will now include environment variables referenced by the container in generated unit files if the value would be looked up from the system environment.
 - Podman now requires that Conmon v2.0.24 be available.
@@ -41,6 +41,8 @@
 ### Bugfixes
 - Fixed a bug where the remote Podman client's `podman build` command did not support the `--arch`, `--platform`, and `--os`, options.
 - Fixed a bug where the remote Podman client's `podman build` command ignored the `--rm=false` option ([#9869](https://github.com/containers/podman/issues/9869)).
+- Fixed a bug where the remote Podman client's `podman build --iidfile` command could include extra output (in addition to just the image ID) in the image ID file written ([#10233](https://github.com/containers/podman/issues/10233)).
+- Fixed a bug where the remote Podman client's `podman build` command did not preserve hardlinks when moving files into the container via `COPY` instructions ([#9893](https://github.com/containers/podman/issues/9893)).
 - Fixed a bug where the `podman generate systemd --new` command could generate extra `--iidfile` arguments if the container was already created with one.
 - Fixed a bug where the `podman generate kube` command produced incorrect YAML for containers which bind-mounted both `/` and `/root` from the host system into the container ([#9764](https://github.com/containers/podman/issues/9764)).
 - Fixed a bug where pods created by `podman play kube` from YAML that specified `ShareProcessNamespace` would only share the PID namespace (and not also the UTS, Network, and IPC namespaces) ([#9128](https://github.com/containers/podman/issues/9128)).
@@ -53,9 +55,9 @@
 - Fixed a bug where setting a custom Seccomp profile via the `seccomp_profile` option in `containers.conf` had no effect, and the default profile was used instead.
 - Fixed a bug where the `--cgroup-parent` option to `podman create` and `podman run` was ignored in rootless Podman on cgroups v2 systems with the `cgroupfs` cgroup manager ([#10173](https://github.com/containers/podman/issues/10173)).
 - Fixed a bug where the `IMAGE` and `NAME` variables in `podman container runlabel` were not being correctly substituted ([#10192](https://github.com/containers/podman/issues/10192)).
-- Fixed a bug where the remote Podman client's `podman build --iidfile` command could include extra output (in addition to just the image ID) in the image ID file written ([#10233](https://github.com/containers/podman/issues/10233)).
 - Fixed a bug where Podman could freeze when creating containers with a specific combination of volumes and working directory ([#10216](https://github.com/containers/podman/issues/10216)).
 - Fixed a bug where rootless Podman containers restarted by restart policy (e.g. containers created with `--restart=always`) would lose networking after being restarted ([#8047](https://github.com/containers/podman/issues/8047)).
+- Fixed a bug where the `podman cp` command could not copy files into containers created with the `--pid=host` flag ([#9985](https://github.com/containers/podman/issues/9985)).
 
 ### API
 - Fixed a bug where the Compat Create endpoint for Containers did not allow advanced network options to be set ([#10110](https://github.com/containers/podman/issues/10110)).