Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

system df: fix image-size calculations #16211

Merged
merged 1 commit into from
Oct 19, 2022
Merged

system df: fix image-size calculations #16211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions cmd/podman/system/df.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,26 +63,29 @@ func df(cmd *cobra.Command, args []string) error {

func printSummary(cmd *cobra.Command, reports *entities.SystemDfReport) error {
var (
dfSummaries []*dfSummary
active int
size, reclaimable int64
dfSummaries []*dfSummary
active int
used int64
)

visitedImages := make(map[string]bool)
for _, i := range reports.Images {
if _, ok := visitedImages[i.ImageID]; ok {
continue
}
visitedImages[i.ImageID] = true
if i.Containers > 0 {
active++
}
size += i.Size
if i.Containers < 1 {
reclaimable += i.Size
used += i.UniqueSize
}
}

imageSummary := dfSummary{
Type: "Images",
Total: len(reports.Images),
Active: active,
RawSize: size,
RawReclaimable: reclaimable,
RawSize: reports.ImagesSize, // The "raw" size is the sum of all layer sizes
RawReclaimable: reports.ImagesSize - used, // We can reclaim the date of "unused" images (i.e., the ones without containers)
}
dfSummaries = append(dfSummaries, &imageSummary)

Expand Down
8 changes: 5 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ require (
github.com/containernetworking/cni v1.1.2
github.com/containernetworking/plugins v1.1.1
github.com/containers/buildah v1.28.0
github.com/containers/common v0.50.2-0.20221014181637-a6a9c521b6a5
github.com/containers/common v0.50.2-0.20221018104440-95c452ec992c
github.com/containers/conmon v2.0.20+incompatible
github.com/containers/image/v5 v5.23.1-0.20221015133641-1921a1993c67
github.com/containers/ocicrypt v1.1.6
github.com/containers/psgo v1.8.0
github.com/containers/storage v1.43.1-0.20221014174544-ebf857fda6f3
github.com/containers/storage v1.43.1-0.20221018232308-33c396d1aacf
github.com/coreos/go-systemd/v22 v22.4.0
github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3
github.com/cyphar/filepath-securejoin v0.2.3
Expand Down Expand Up @@ -79,7 +79,7 @@ require (
github.com/chzyer/readline v1.5.1 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/containerd/containerd v1.6.8 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect
github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
Expand Down Expand Up @@ -147,3 +147,5 @@ require (
)

replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.1-0.20220617142545-8b9452f75cbc

replace github.com/containers/storage => github.com/containers/storage v1.43.1-0.20221014174544-ebf857fda6f3
16 changes: 4 additions & 12 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,6 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX
github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
Expand All @@ -142,7 +141,6 @@ github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn
github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0=
github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
Expand Down Expand Up @@ -377,9 +375,9 @@ github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFY
github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM=
github.com/containerd/stargz-snapshotter/estargz v0.9.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0=
github.com/containerd/stargz-snapshotter/estargz v0.12.0 h1:idtwRTLjk2erqiYhPWy2L844By8NRFYEwYHcXhoIWPM=
github.com/containerd/stargz-snapshotter/estargz v0.12.0/go.mod h1:AIQ59TewBFJ4GOPEQXujcrJ/EKxh5xXZegW1rkR1P/M=
github.com/containerd/stargz-snapshotter/estargz v0.12.1 h1:+7nYmHJb0tEkcRaAW+MHqoKaJYZmkikupxCqVtmPuY0=
github.com/containerd/stargz-snapshotter/estargz v0.12.1/go.mod h1:12VUuCq3qPq4y8yUW+l5w3+oXV3cx2Po3KSe/SmPGqw=
github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
Expand Down Expand Up @@ -410,12 +408,11 @@ github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19
github.com/containers/buildah v1.28.0 h1:63Kpf9nAEJGsDEOArb5Q0dn5S3B9wFQc9kST4nU7+Pw=
github.com/containers/buildah v1.28.0/go.mod h1:pTYSfpf+Ha/KbnMmwhhhEjkSF3NuhpxZfiDNDORLgqY=
github.com/containers/common v0.50.1/go.mod h1:XnWlXPyE9Ky+8v8MfYWJZFnejkprAkUeo0DTWmSiwcY=
github.com/containers/common v0.50.2-0.20221014181637-a6a9c521b6a5 h1:Sp6lus6zXyg+Fv3ecoPOXm+80xC23YrKtntqYQU1pCA=
github.com/containers/common v0.50.2-0.20221014181637-a6a9c521b6a5/go.mod h1:7AM52N4n4UHzAeArB1vmIw7NhDNpDxrFaycubysYgFo=
github.com/containers/common v0.50.2-0.20221018104440-95c452ec992c h1:ZFMv4o5Mx8iq2O0AuE9sH4uiKqYggkdJPsz3KY65aGg=
github.com/containers/common v0.50.2-0.20221018104440-95c452ec992c/go.mod h1:71Q12SyFJO8rWgdMjvR6MC73KGpIpQU6PPkBRgoSots=
github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
github.com/containers/image/v5 v5.23.0/go.mod h1:EXFFGEsL99S6aqLqK2mQJ3yrNh6Q05UCHt4mhF9JNoM=
github.com/containers/image/v5 v5.23.1-0.20221013202101-87afcefe9766/go.mod h1:2JJxA5K1NFpA3FtrK+Csmdlj++5oveB7CsXhekEJsIU=
github.com/containers/image/v5 v5.23.1-0.20221015133641-1921a1993c67 h1:kCKq/UASl4o6mwlqbBP8r+6fMIdQg3RkC2znw6H65A4=
github.com/containers/image/v5 v5.23.1-0.20221015133641-1921a1993c67/go.mod h1:V+wF8uIye+czYC4HWHGh4IK9dZBTsQUG1ZYie9gXxCQ=
github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a h1:spAGlqziZjCJL25C6F1zsQY05tfCKE9F5YwtEWWe6hU=
Expand All @@ -430,9 +427,6 @@ github.com/containers/ocicrypt v1.1.6 h1:uoG52u2e91RE4UqmBICZY8dNshgfvkdl3BW6jnx
github.com/containers/ocicrypt v1.1.6/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc=
github.com/containers/psgo v1.8.0 h1:2loGekmGAxM9ir5OsXWEfGwFxorMPYnc6gEDsGFQvhY=
github.com/containers/psgo v1.8.0/go.mod h1:T8ZxnX3Ur4RvnhxFJ7t8xJ1F48RhiZB4rSrOaR/qGHc=
github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c+Q/45RlH6r4=
github.com/containers/storage v1.43.0/go.mod h1:uZ147thiIFGdVTjMmIw19knttQnUCl3y9zjreHrg11s=
github.com/containers/storage v1.43.1-0.20221014072257-a144fee6f51c/go.mod h1:K2qol6lCT/LRqZ3TMNRBU22tCTC6/Mb4G23K5SHhrYw=
github.com/containers/storage v1.43.1-0.20221014174544-ebf857fda6f3 h1:3jBfOfHFmaubuvhKSgfV7Krr2ZNpOlGiU9OhQYkoTxY=
github.com/containers/storage v1.43.1-0.20221014174544-ebf857fda6f3/go.mod h1:K2qol6lCT/LRqZ3TMNRBU22tCTC6/Mb4G23K5SHhrYw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
Expand Down Expand Up @@ -1411,7 +1405,6 @@ github.com/opencontainers/runtime-tools v0.9.1-0.20221014010322-58c91d646d86/go.
github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
github.com/opencontainers/selinux v1.8.5/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo=
github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
Expand Down Expand Up @@ -2154,7 +2147,6 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
1 change: 1 addition & 0 deletions pkg/domain/entities/system.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ type SystemDfOptions struct {

// SystemDfReport describes the response for df information
type SystemDfReport struct {
ImagesSize int64
Images []*SystemDfImageReport
Containers []*SystemDfContainerReport
Volumes []*SystemDfVolumeReport
Expand Down
4 changes: 3 additions & 1 deletion pkg/domain/infra/abi/system.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ func (ic *ContainerEngine) SystemDf(ctx context.Context, options entities.System
dfImages = []*entities.SystemDfImageReport{}
)

imageStats, err := ic.Libpod.LibimageRuntime().DiskUsage(ctx)
imageStats, totalImageSize, err := ic.Libpod.LibimageRuntime().DiskUsage(ctx)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -352,7 +352,9 @@ func (ic *ContainerEngine) SystemDf(ctx context.Context, options entities.System
}
dfVolumes = append(dfVolumes, &report)
}

return &entities.SystemDfReport{
ImagesSize: totalImageSize,
Images: dfImages,
Containers: dfContainers,
Volumes: dfVolumes,
Expand Down
2 changes: 1 addition & 1 deletion test/apiv2/45-system.at
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ t POST 'libpod/system/prune?volumes=true&all=true' params='' 200

## podman system df
t GET system/df 200 '{"LayersSize":0,"Images":[],"Containers":[],"Volumes":[],"BuildCache":[],"BuilderSize":0}'
t GET libpod/system/df 200 '{"Images":[],"Containers":[],"Volumes":[]}'
t GET libpod/system/df 200 '{"ImagesSize":0,"Images":[],"Containers":[],"Volumes":[]}'

# Create volume. We expect df to report this volume next invocation of system/df
t GET libpod/info 200
Expand Down
31 changes: 30 additions & 1 deletion test/system/320-system-df.bats
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,39 @@ function teardown() {
is "${lines[10]}" "[0-9a-f]\{64\} *[01] * 0B" \
"system df -v, 'Volumes', second line"

# Clean up
# Make sure that the return image "raw" size is correct
run_podman image inspect $IMAGE --format "{{.Size}}"
expectedSize="$output"

run_podman system df --format "{{.RawSize}}"
is "${lines[0]}" "$expectedSize" "raw image size is correct"

# Clean up and check reclaimable image data
run_podman system df --format '{{.Reclaimable}}'
is "${lines[0]}" "0B (0%)" "cannot reclaim image data as it's still used by the containers"

run_podman exec c2 touch /stop
run_podman wait c2

# Create a second image by committing a container.
run_podman container commit -q c1
image="$output"

run_podman system df --format '{{.Reclaimable}}'
is "${lines[0]}" ".* (100%)" "100 percent of image data is reclaimable because $IMAGE has unique size of 0"

# Make sure the unique size is now really 0. We cannot use --format for
# that unfortunately but we can exploit the fact that $IMAGE is used by
# two containers.
run_podman system df -v
is "$output" ".*0B\\s\\+2.*"

run_podman rm c1 c2

run_podman system df --format '{{.Reclaimable}}'
is "${lines[0]}" ".* (100%)" "100 percent of image data is reclaimable because all containers are gone"

run_podman rmi $image
run_podman volume rm -a
}

Expand Down
35 changes: 30 additions & 5 deletions vendor/github.com/containers/common/libimage/disk_usage.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion vendor/github.com/containers/common/libimage/filters.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions vendor/github.com/containers/common/libimage/image.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions vendor/github.com/containers/common/libimage/layer_tree.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions vendor/github.com/containers/common/libimage/platform.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 1 addition & 2 deletions vendor/github.com/containers/common/libnetwork/cni/cni_conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 1 addition & 2 deletions vendor/github.com/containers/common/libnetwork/netavark/network.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions vendor/github.com/containers/common/libnetwork/network/interface.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading