Merge pull request #16211 from vrothberg/fix-16135

system df: fix image-size calculations

cmd/podman/system/df.go

@@ -63,26 +63,29 @@ func df(cmd *cobra.Command, args []string) error {
 
 func printSummary(cmd *cobra.Command, reports *entities.SystemDfReport) error {
 	var (
-		dfSummaries       []*dfSummary
-		active            int
-		size, reclaimable int64
+		dfSummaries []*dfSummary
+		active      int
+		used        int64
 	)
 
+	visitedImages := make(map[string]bool)
 	for _, i := range reports.Images {
+		if _, ok := visitedImages[i.ImageID]; ok {
+			continue
+		}
+		visitedImages[i.ImageID] = true
 		if i.Containers > 0 {
 			active++
-		}
-		size += i.Size
-		if i.Containers < 1 {
-			reclaimable += i.Size
+			used += i.UniqueSize
 		}
 	}
+
 	imageSummary := dfSummary{
 		Type:           "Images",
 		Total:          len(reports.Images),
 		Active:         active,
-		RawSize:        size,
-		RawReclaimable: reclaimable,
+		RawSize:        reports.ImagesSize,        // The "raw" size is the sum of all layer sizes
+		RawReclaimable: reports.ImagesSize - used, // We can reclaim the date of "unused" images (i.e., the ones without containers)
 	}
 	dfSummaries = append(dfSummaries, &imageSummary)
 

go.mod

@@ -12,12 +12,12 @@ require (
 	github.com/containernetworking/cni v1.1.2
 	github.com/containernetworking/plugins v1.1.1
 	github.com/containers/buildah v1.28.0
-	github.com/containers/common v0.50.2-0.20221014181637-a6a9c521b6a5
+	github.com/containers/common v0.50.2-0.20221018104440-95c452ec992c
 	github.com/containers/conmon v2.0.20+incompatible
 	github.com/containers/image/v5 v5.23.1-0.20221015133641-1921a1993c67
 	github.com/containers/ocicrypt v1.1.6
 	github.com/containers/psgo v1.8.0
-	github.com/containers/storage v1.43.1-0.20221014174544-ebf857fda6f3
+	github.com/containers/storage v1.43.1-0.20221018232308-33c396d1aacf
 	github.com/coreos/go-systemd/v22 v22.4.0
 	github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3
 	github.com/cyphar/filepath-securejoin v0.2.3
@@ -79,7 +79,7 @@ require (
 	github.com/chzyer/readline v1.5.1 // indirect
 	github.com/containerd/cgroups v1.0.4 // indirect
 	github.com/containerd/containerd v1.6.8 // indirect
-	github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect
 	github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -147,3 +147,5 @@ require (
 )
 
 replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.1-0.20220617142545-8b9452f75cbc
+
+replace github.com/containers/storage => github.com/containers/storage v1.43.1-0.20221014174544-ebf857fda6f3

go.sum

@@ -129,7 +129,6 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX
 github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
@@ -142,7 +141,6 @@ github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn
 github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
 github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
 github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
-github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0=
 github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
 github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
 github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
@@ -377,9 +375,9 @@ github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFY
 github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM=
-github.com/containerd/stargz-snapshotter/estargz v0.9.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0=
-github.com/containerd/stargz-snapshotter/estargz v0.12.0 h1:idtwRTLjk2erqiYhPWy2L844By8NRFYEwYHcXhoIWPM=
 github.com/containerd/stargz-snapshotter/estargz v0.12.0/go.mod h1:AIQ59TewBFJ4GOPEQXujcrJ/EKxh5xXZegW1rkR1P/M=
+github.com/containerd/stargz-snapshotter/estargz v0.12.1 h1:+7nYmHJb0tEkcRaAW+MHqoKaJYZmkikupxCqVtmPuY0=
+github.com/containerd/stargz-snapshotter/estargz v0.12.1/go.mod h1:12VUuCq3qPq4y8yUW+l5w3+oXV3cx2Po3KSe/SmPGqw=
 github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
@@ -410,12 +408,11 @@ github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19
 github.com/containers/buildah v1.28.0 h1:63Kpf9nAEJGsDEOArb5Q0dn5S3B9wFQc9kST4nU7+Pw=
 github.com/containers/buildah v1.28.0/go.mod h1:pTYSfpf+Ha/KbnMmwhhhEjkSF3NuhpxZfiDNDORLgqY=
 github.com/containers/common v0.50.1/go.mod h1:XnWlXPyE9Ky+8v8MfYWJZFnejkprAkUeo0DTWmSiwcY=
-github.com/containers/common v0.50.2-0.20221014181637-a6a9c521b6a5 h1:Sp6lus6zXyg+Fv3ecoPOXm+80xC23YrKtntqYQU1pCA=
-github.com/containers/common v0.50.2-0.20221014181637-a6a9c521b6a5/go.mod h1:7AM52N4n4UHzAeArB1vmIw7NhDNpDxrFaycubysYgFo=
+github.com/containers/common v0.50.2-0.20221018104440-95c452ec992c h1:ZFMv4o5Mx8iq2O0AuE9sH4uiKqYggkdJPsz3KY65aGg=
+github.com/containers/common v0.50.2-0.20221018104440-95c452ec992c/go.mod h1:71Q12SyFJO8rWgdMjvR6MC73KGpIpQU6PPkBRgoSots=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
 github.com/containers/image/v5 v5.23.0/go.mod h1:EXFFGEsL99S6aqLqK2mQJ3yrNh6Q05UCHt4mhF9JNoM=
-github.com/containers/image/v5 v5.23.1-0.20221013202101-87afcefe9766/go.mod h1:2JJxA5K1NFpA3FtrK+Csmdlj++5oveB7CsXhekEJsIU=
 github.com/containers/image/v5 v5.23.1-0.20221015133641-1921a1993c67 h1:kCKq/UASl4o6mwlqbBP8r+6fMIdQg3RkC2znw6H65A4=
 github.com/containers/image/v5 v5.23.1-0.20221015133641-1921a1993c67/go.mod h1:V+wF8uIye+czYC4HWHGh4IK9dZBTsQUG1ZYie9gXxCQ=
 github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a h1:spAGlqziZjCJL25C6F1zsQY05tfCKE9F5YwtEWWe6hU=
@@ -430,9 +427,6 @@ github.com/containers/ocicrypt v1.1.6 h1:uoG52u2e91RE4UqmBICZY8dNshgfvkdl3BW6jnx
 github.com/containers/ocicrypt v1.1.6/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc=
 github.com/containers/psgo v1.8.0 h1:2loGekmGAxM9ir5OsXWEfGwFxorMPYnc6gEDsGFQvhY=
 github.com/containers/psgo v1.8.0/go.mod h1:T8ZxnX3Ur4RvnhxFJ7t8xJ1F48RhiZB4rSrOaR/qGHc=
-github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c+Q/45RlH6r4=
-github.com/containers/storage v1.43.0/go.mod h1:uZ147thiIFGdVTjMmIw19knttQnUCl3y9zjreHrg11s=
-github.com/containers/storage v1.43.1-0.20221014072257-a144fee6f51c/go.mod h1:K2qol6lCT/LRqZ3TMNRBU22tCTC6/Mb4G23K5SHhrYw=
 github.com/containers/storage v1.43.1-0.20221014174544-ebf857fda6f3 h1:3jBfOfHFmaubuvhKSgfV7Krr2ZNpOlGiU9OhQYkoTxY=
 github.com/containers/storage v1.43.1-0.20221014174544-ebf857fda6f3/go.mod h1:K2qol6lCT/LRqZ3TMNRBU22tCTC6/Mb4G23K5SHhrYw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -1411,7 +1405,6 @@ github.com/opencontainers/runtime-tools v0.9.1-0.20221014010322-58c91d646d86/go.
 github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
-github.com/opencontainers/selinux v1.8.5/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo=
 github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
@@ -2154,7 +2147,6 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

pkg/domain/entities/system.go

@@ -47,6 +47,7 @@ type SystemDfOptions struct {
 
 // SystemDfReport describes the response for df information
 type SystemDfReport struct {
+	ImagesSize int64
 	Images     []*SystemDfImageReport
 	Containers []*SystemDfContainerReport
 	Volumes    []*SystemDfVolumeReport

pkg/domain/infra/abi/system.go

@@ -253,7 +253,7 @@ func (ic *ContainerEngine) SystemDf(ctx context.Context, options entities.System
 		dfImages = []*entities.SystemDfImageReport{}
 	)
 
-	imageStats, err := ic.Libpod.LibimageRuntime().DiskUsage(ctx)
+	imageStats, totalImageSize, err := ic.Libpod.LibimageRuntime().DiskUsage(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -352,7 +352,9 @@ func (ic *ContainerEngine) SystemDf(ctx context.Context, options entities.System
 		}
 		dfVolumes = append(dfVolumes, &report)
 	}
+
 	return &entities.SystemDfReport{
+		ImagesSize: totalImageSize,
 		Images:     dfImages,
 		Containers: dfContainers,
 		Volumes:    dfVolumes,

test/apiv2/45-system.at

@@ -8,7 +8,7 @@ t POST 'libpod/system/prune?volumes=true&all=true' params='' 200
 
 ## podman system df
 t GET system/df 200 '{"LayersSize":0,"Images":[],"Containers":[],"Volumes":[],"BuildCache":[],"BuilderSize":0}'
-t GET libpod/system/df 200 '{"Images":[],"Containers":[],"Volumes":[]}'
+t GET libpod/system/df 200 '{"ImagesSize":0,"Images":[],"Containers":[],"Volumes":[]}'
 
 # Create volume. We expect df to report this volume next invocation of system/df
 t GET libpod/info 200

test/system/320-system-df.bats

@@ -52,10 +52,39 @@ function teardown() {
     is "${lines[10]}" "[0-9a-f]\{64\} *[01] * 0B" \
        "system df -v, 'Volumes', second line"
 
-    # Clean up
+    # Make sure that the return image "raw" size is correct
+    run_podman image inspect $IMAGE --format "{{.Size}}"
+    expectedSize="$output"
+
+    run_podman system df --format "{{.RawSize}}"
+    is "${lines[0]}" "$expectedSize" "raw image size is correct"
+
+    # Clean up and check reclaimable image data
+    run_podman system df --format '{{.Reclaimable}}'
+    is "${lines[0]}" "0B (0%)" "cannot reclaim image data as it's still used by the containers"
+
     run_podman exec c2 touch /stop
     run_podman wait c2
+
+    # Create a second image by committing a container.
+    run_podman container commit -q c1
+    image="$output"
+
+    run_podman system df --format '{{.Reclaimable}}'
+    is "${lines[0]}" ".* (100%)" "100 percent of image data is reclaimable because $IMAGE has unique size of 0"
+
+    # Make sure the unique size is now really 0.  We cannot use --format for
+    # that unfortunately but we can exploit the fact that $IMAGE is used by
+    # two containers.
+    run_podman system df -v
+    is "$output" ".*0B\\s\\+2.*"
+
     run_podman rm c1 c2
+
+    run_podman system df --format '{{.Reclaimable}}'
+    is "${lines[0]}" ".* (100%)" "100 percent of image data is reclaimable because all containers are gone"
+
+    run_podman rmi $image
     run_podman volume rm -a
 }