diff --git a/cmd/podman/common/specgen.go b/cmd/podman/common/specgen.go
index 975c76fd93..eff8b43aa7 100644
--- a/cmd/podman/common/specgen.go
+++ b/cmd/podman/common/specgen.go
@@ -148,17 +148,16 @@ func getMemoryLimits(s *specgen.SpecGenerator, c *ContainerCLIOpts) (*specs.Linu
 	}
 	if m := c.MemorySwap; len(m) > 0 {
 		var ms int64
-		if m == "-1" {
-			ms = int64(-1)
-			s.ResourceLimits.Memory.Swap = &ms
-		} else {
+		// only set memory swap if it was set
+		// -1 indicates unlimited
+		if m != "-1" {
 			ms, err = units.RAMInBytes(m)
+			memory.Swap = &ms
 			if err != nil {
 				return nil, errors.Wrapf(err, "invalid value for memory")
 			}
+			hasLimits = true
 		}
-		memory.Swap = &ms
-		hasLimits = true
 	}
 	if m := c.KernelMemory; len(m) > 0 {
 		mk, err := units.RAMInBytes(m)
diff --git a/test/e2e/run_memory_test.go b/test/e2e/run_memory_test.go
index ad3a2b54fd..8371d3caec 100644
--- a/test/e2e/run_memory_test.go
+++ b/test/e2e/run_memory_test.go
@@ -2,6 +2,7 @@ package integration
 
 import (
 	"os"
+	"strconv"
 
 	. "github.com/containers/podman/v2/test/utils"
 	. "github.com/onsi/ginkgo"
@@ -90,4 +91,27 @@ var _ = Describe("Podman run memory", func() {
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.OutputToString()).To(Equal("41943040"))
 	})
+
+	It("podman run kernel-memory test", func() {
+		if podmanTest.Host.Distribution == "ubuntu" {
+			Skip("Unable to perform test on Ubuntu distributions due to memory management")
+		}
+		var session *PodmanSessionIntegration
+		if CGROUPSV2 {
+			session = podmanTest.Podman([]string{"run", "--memory", "256m", "--memory-swap", "-1", ALPINE, "cat", "/sys/fs/cgroup/memory.swap.max"})
+		} else {
+			session = podmanTest.Podman([]string{"run", "--cgroupns=private", ALPINE, "cat", "/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes"})
+		}
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(BeZero())
+		output := session.OutputToString()
+		Expect(err).To(BeNil())
+		if CGROUPSV2 {
+			Expect(output).To(Equal("max"))
+		} else {
+			crazyHighNumber, err := strconv.ParseInt(output, 10, 64)
+			Expect(err).To(BeZero())
+			Expect(crazyHighNumber).To(BeNumerically(">", 936854771712))
+		}
+	})
 })