From 459248337ba7a6a82a76970a6711d07fd3f6fdba Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 21 Jun 2023 14:16:20 +0200 Subject: [PATCH 1/2] vendor: bump containers/common Signed-off-by: Giuseppe Scrivano --- go.mod | 2 +- go.sum | 4 ++-- vendor/github.com/containers/common/pkg/config/config.go | 3 +++ .../containers/common/pkg/config/containers.conf | 6 ++++++ .../containers/common/pkg/config/containers.conf-freebsd | 7 +++++++ vendor/modules.txt | 2 +- 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index d895507c70..db8001749e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.3.0 github.com/containers/buildah v1.30.1-0.20230504052500-e925b5852e07 - github.com/containers/common v0.53.1-0.20230620132900-ac2475afa81d + github.com/containers/common v0.53.1-0.20230621115248-a2cd3ea30337 github.com/containers/conmon v2.0.20+incompatible github.com/containers/image/v5 v5.25.1-0.20230613183705-07ced6137083 github.com/containers/libhvee v0.0.5 diff --git a/go.sum b/go.sum index 923739b786..035f51d5ab 100644 --- a/go.sum +++ b/go.sum @@ -239,8 +239,8 @@ github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0= github.com/containers/buildah v1.30.1-0.20230504052500-e925b5852e07 h1:Bs2sNFh/fSYr4J6JJLFqzyn3dp6HhlA6ewFwRYUpeIE= github.com/containers/buildah v1.30.1-0.20230504052500-e925b5852e07/go.mod h1:6A/BK0YJLXL8+AqlbceKJrhUT+NtEgsvAc51F7TAllc= -github.com/containers/common v0.53.1-0.20230620132900-ac2475afa81d h1:Z+xdHWSwjW/VGdYGufKfqji+G7FQ1IdkFai0MOpqzd4= -github.com/containers/common v0.53.1-0.20230620132900-ac2475afa81d/go.mod h1:qE1MzGl69IoK7ZNCCH51+aLVjyQtnH0LiZe0wG32Jy0= +github.com/containers/common v0.53.1-0.20230621115248-a2cd3ea30337 h1:Z9wxp08tzCKgI3ziVwpoMyQcDKH8z9VmgyeHJcnunj4= +github.com/containers/common v0.53.1-0.20230621115248-a2cd3ea30337/go.mod h1:qE1MzGl69IoK7ZNCCH51+aLVjyQtnH0LiZe0wG32Jy0= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/image/v5 v5.25.1-0.20230613183705-07ced6137083 h1:6Pbnll97ls6G0U3DSxaTqp7Sd8Fykc4gd7BUJm7Bpn8= diff --git a/vendor/github.com/containers/common/pkg/config/config.go b/vendor/github.com/containers/common/pkg/config/config.go index 3ed71f662e..a408b4fd48 100644 --- a/vendor/github.com/containers/common/pkg/config/config.go +++ b/vendor/github.com/containers/common/pkg/config/config.go @@ -510,6 +510,9 @@ type EngineConfig struct { // CompressionFormat is the compression format used to compress image layers. CompressionFormat string `toml:"compression_format,omitempty"` + + // CompressionLevel is the compression level used to compress image layers. + CompressionLevel *int `toml:"compression_level,omitempty"` } // SetOptions contains a subset of options in a Config. It's used to indicate if diff --git a/vendor/github.com/containers/common/pkg/config/containers.conf b/vendor/github.com/containers/common/pkg/config/containers.conf index c4efc41489..ec881f2fc2 100644 --- a/vendor/github.com/containers/common/pkg/config/containers.conf +++ b/vendor/github.com/containers/common/pkg/config/containers.conf @@ -381,6 +381,12 @@ default_sysctls = [ # #compression_format = "gzip" +# The compression level to use when pushing an image. +# Valid options depend on the compression format used. +# For gzip, valid options are 1-9, with a default of 5. +# For zstd, valid options are 1-20, with a default of 3. +# +#compression_level = 5 # Cgroup management implementation used for the runtime. # Valid options "systemd" or "cgroupfs" diff --git a/vendor/github.com/containers/common/pkg/config/containers.conf-freebsd b/vendor/github.com/containers/common/pkg/config/containers.conf-freebsd index 7fe7538a1a..5e187893b2 100644 --- a/vendor/github.com/containers/common/pkg/config/containers.conf-freebsd +++ b/vendor/github.com/containers/common/pkg/config/containers.conf-freebsd @@ -311,6 +311,13 @@ default_sysctls = [ # #compression_format = "gzip" +# The compression level to use when pushing an image. +# Valid options depend on the compression format used. +# For gzip, valid options are 1-9, with a default of 5. +# For zstd, valid options are 1-20, with a default of 3. +# +#compression_level = 5 + # Environment variables to pass into conmon # #conmon_env_vars = [ diff --git a/vendor/modules.txt b/vendor/modules.txt index 78d2ab513c..6fc4d7e393 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -125,7 +125,7 @@ github.com/containers/buildah/pkg/rusage github.com/containers/buildah/pkg/sshagent github.com/containers/buildah/pkg/util github.com/containers/buildah/util -# github.com/containers/common v0.53.1-0.20230620132900-ac2475afa81d +# github.com/containers/common v0.53.1-0.20230621115248-a2cd3ea30337 ## explicit; go 1.18 github.com/containers/common/libimage github.com/containers/common/libimage/define From bb932cc840a97a5b365ffcf05d9679d2954201ee Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Tue, 20 Jun 2023 11:10:11 +0200 Subject: [PATCH 2/2] cmd, push: expose --compression-level This patch adds the --compression-level option to the push command. Closes: https://github.com/containers/podman/issues/18939 Signed-off-by: Giuseppe Scrivano --- cmd/podman/images/push.go | 12 +++++++++ cmd/podman/manifest/push.go | 13 +++++++++ .../markdown/options/compression-level.md | 7 +++++ .../markdown/podman-manifest-push.1.md.in | 2 ++ docs/source/markdown/podman-push.1.md.in | 2 ++ pkg/api/handlers/libpod/images_push.go | 2 ++ pkg/api/handlers/libpod/manifests.go | 2 ++ pkg/bindings/images/types.go | 2 ++ pkg/bindings/images/types_push_options.go | 15 +++++++++++ pkg/domain/entities/images.go | 2 ++ pkg/domain/infra/abi/images.go | 9 +++++++ pkg/domain/infra/abi/manifest.go | 8 ++++++ pkg/domain/infra/tunnel/images.go | 4 +++ test/e2e/manifest_test.go | 27 ++++++++++++++++--- test/e2e/push_test.go | 14 +++++++--- 15 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 docs/source/markdown/options/compression-level.md diff --git a/cmd/podman/images/push.go b/cmd/podman/images/push.go index 490fb6b406..f0858aee42 100644 --- a/cmd/podman/images/push.go +++ b/cmd/podman/images/push.go @@ -130,6 +130,10 @@ func pushFlags(cmd *cobra.Command) { flags.StringVar(&pushOptions.CompressionFormat, compressionFormat, "", "compression format to use") _ = cmd.RegisterFlagCompletionFunc(compressionFormat, common.AutocompleteCompressionFormat) + compressionLevel := "compression-level" + flags.Int(compressionLevel, 0, "compression level to use") + _ = cmd.RegisterFlagCompletionFunc(compressionLevel, completion.AutocompleteNone) + encryptionKeysFlagName := "encryption-key" flags.StringSliceVar(&pushOptions.EncryptionKeys, encryptionKeysFlagName, nil, "Key with the encryption protocol to use to encrypt the image (e.g. jwe:/path/to/key.pem)") _ = cmd.RegisterFlagCompletionFunc(encryptionKeysFlagName, completion.AutocompleteDefault) @@ -201,6 +205,14 @@ func imagePush(cmd *cobra.Command, args []string) error { pushOptions.OciEncryptConfig = encConfig pushOptions.OciEncryptLayers = encLayers + if cmd.Flags().Changed("compression-level") { + val, err := cmd.Flags().GetInt("compression-level") + if err != nil { + return err + } + pushOptions.CompressionLevel = &val + } + // Let's do all the remaining Yoga in the API to prevent us from scattering // logic across (too) many parts of the code. report, err := registry.ImageEngine().Push(registry.GetContext(), source, destination, pushOptions.ImagePushOptions) diff --git a/cmd/podman/manifest/push.go b/cmd/podman/manifest/push.go index e61170333f..24d0a7eebd 100644 --- a/cmd/podman/manifest/push.go +++ b/cmd/podman/manifest/push.go @@ -100,6 +100,10 @@ func init() { flags.StringVar(&manifestPushOpts.CompressionFormat, compressionFormat, "", "compression format to use") _ = pushCmd.RegisterFlagCompletionFunc(compressionFormat, common.AutocompleteCompressionFormat) + compressionLevel := "compression-level" + flags.Int(compressionLevel, 0, "compression level to use") + _ = pushCmd.RegisterFlagCompletionFunc(compressionLevel, completion.AutocompleteNone) + if registry.IsRemote() { _ = flags.MarkHidden("cert-dir") _ = flags.MarkHidden(signByFlagName) @@ -155,6 +159,15 @@ func push(cmd *cobra.Command, args []string) error { } manifestPushOpts.SkipTLSVerify = types.NewOptionalBool(manifestPushOpts.Insecure) } + + if cmd.Flags().Changed("compression-level") { + val, err := cmd.Flags().GetInt("compression-level") + if err != nil { + return err + } + manifestPushOpts.CompressionLevel = &val + } + digest, err := registry.ImageEngine().ManifestPush(registry.Context(), listImageSpec, destSpec, manifestPushOpts.ImagePushOptions) if err != nil { return err diff --git a/docs/source/markdown/options/compression-level.md b/docs/source/markdown/options/compression-level.md new file mode 100644 index 0000000000..f20f4a2a28 --- /dev/null +++ b/docs/source/markdown/options/compression-level.md @@ -0,0 +1,7 @@ +####> This option file is used in: +####> podman manifest push, push +####> If file is edited, make sure the changes +####> are applicable to all of those. +#### **--compression-level**=*level* + +Specifies the compression level to use. The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive) with a default of 3, while for gzip it is 1-9 (inclusive) and has a default of 5. diff --git a/docs/source/markdown/podman-manifest-push.1.md.in b/docs/source/markdown/podman-manifest-push.1.md.in index 3064dd1558..97f899b8a8 100644 --- a/docs/source/markdown/podman-manifest-push.1.md.in +++ b/docs/source/markdown/podman-manifest-push.1.md.in @@ -25,6 +25,8 @@ the list or index itself. (Default true) @@option compression-format +@@option compression-level + @@option creds @@option digestfile diff --git a/docs/source/markdown/podman-push.1.md.in b/docs/source/markdown/podman-push.1.md.in index 3a5401282a..58bf1afe8c 100644 --- a/docs/source/markdown/podman-push.1.md.in +++ b/docs/source/markdown/podman-push.1.md.in @@ -54,6 +54,8 @@ Note: This flag can only be set when using the **dir** transport @@option compression-format +@@option compression-level + @@option creds @@option digestfile diff --git a/pkg/api/handlers/libpod/images_push.go b/pkg/api/handlers/libpod/images_push.go index 1241a07759..4bd8393d5d 100644 --- a/pkg/api/handlers/libpod/images_push.go +++ b/pkg/api/handlers/libpod/images_push.go @@ -27,6 +27,7 @@ func PushImage(w http.ResponseWriter, r *http.Request) { query := struct { All bool `schema:"all"` CompressionFormat string `schema:"compressionFormat"` + CompressionLevel *int `schema:"compressionLevel"` Destination string `schema:"destination"` Format string `schema:"format"` RemoveSignatures bool `schema:"removeSignatures"` @@ -75,6 +76,7 @@ func PushImage(w http.ResponseWriter, r *http.Request) { All: query.All, Authfile: authfile, CompressionFormat: query.CompressionFormat, + CompressionLevel: query.CompressionLevel, Format: query.Format, Password: password, Quiet: true, diff --git a/pkg/api/handlers/libpod/manifests.go b/pkg/api/handlers/libpod/manifests.go index 0702facb3a..bc0f1c628c 100644 --- a/pkg/api/handlers/libpod/manifests.go +++ b/pkg/api/handlers/libpod/manifests.go @@ -328,6 +328,7 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { query := struct { All bool `schema:"all"` CompressionFormat string `schema:"compressionFormat"` + CompressionLevel *int `schema:"compressionLevel"` Format string `schema:"format"` RemoveSignatures bool `schema:"removeSignatures"` TLSVerify bool `schema:"tlsVerify"` @@ -366,6 +367,7 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { All: query.All, Authfile: authfile, CompressionFormat: query.CompressionFormat, + CompressionLevel: query.CompressionLevel, Format: query.Format, Password: password, Quiet: true, diff --git a/pkg/bindings/images/types.go b/pkg/bindings/images/types.go index 36fc06ffcf..9c3ed2b108 100644 --- a/pkg/bindings/images/types.go +++ b/pkg/bindings/images/types.go @@ -142,6 +142,8 @@ type PushOptions struct { Compress *bool // CompressionFormat is the format to use for the compression of the blobs CompressionFormat *string + // CompressionLevel is the level to use for the compression of the blobs + CompressionLevel *int // Manifest type of the pushed image Format *string // Password for authenticating against the registry. diff --git a/pkg/bindings/images/types_push_options.go b/pkg/bindings/images/types_push_options.go index 71f3d9e3eb..550b1f4070 100644 --- a/pkg/bindings/images/types_push_options.go +++ b/pkg/bindings/images/types_push_options.go @@ -78,6 +78,21 @@ func (o *PushOptions) GetCompressionFormat() string { return *o.CompressionFormat } +// WithCompressionLevel set field CompressionLevel to given value +func (o *PushOptions) WithCompressionLevel(value int) *PushOptions { + o.CompressionLevel = &value + return o +} + +// GetCompressionLevel returns value of field CompressionLevel +func (o *PushOptions) GetCompressionLevel() int { + if o.CompressionLevel == nil { + var z int + return z + } + return *o.CompressionLevel +} + // WithFormat set field Format to given value func (o *PushOptions) WithFormat(value string) *PushOptions { o.Format = &value diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index 69bf748d83..c58594d983 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -231,6 +231,8 @@ type ImagePushOptions struct { Progress chan types.ProgressProperties // CompressionFormat is the format to use for the compression of the blobs CompressionFormat string + // CompressionLevel is the level to use for the compression of the blobs + CompressionLevel *int // Writer is used to display copy information including progress bars. Writer io.Writer // OciEncryptConfig when non-nil indicates that an image should be encrypted. diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index 0ba9471c2e..5b22158dd7 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -316,6 +316,7 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri pushOptions.Writer = options.Writer pushOptions.OciEncryptConfig = options.OciEncryptConfig pushOptions.OciEncryptLayers = options.OciEncryptLayers + pushOptions.CompressionLevel = options.CompressionLevel compressionFormat := options.CompressionFormat if compressionFormat == "" { @@ -333,6 +334,14 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri pushOptions.CompressionFormat = &algo } + if pushOptions.CompressionLevel == nil { + config, err := ir.Libpod.GetConfigNoCopy() + if err != nil { + return nil, err + } + pushOptions.CompressionLevel = config.Engine.CompressionLevel + } + if !options.Quiet && pushOptions.Writer == nil { pushOptions.Writer = os.Stderr } diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index 1c894891be..bc60e231ef 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -340,6 +340,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin pushOptions.SignSigstorePrivateKeyPassphrase = opts.SignSigstorePrivateKeyPassphrase pushOptions.InsecureSkipTLSVerify = opts.SkipTLSVerify pushOptions.Writer = opts.Writer + pushOptions.CompressionLevel = opts.CompressionLevel compressionFormat := opts.CompressionFormat if compressionFormat == "" { @@ -356,6 +357,13 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin } pushOptions.CompressionFormat = &algo } + if pushOptions.CompressionLevel == nil { + config, err := ir.Libpod.GetConfigNoCopy() + if err != nil { + return "", err + } + pushOptions.CompressionLevel = config.Engine.CompressionLevel + } if opts.All { pushOptions.ImageListSelection = cp.CopyAllImages diff --git a/pkg/domain/infra/tunnel/images.go b/pkg/domain/infra/tunnel/images.go index e2e6a24511..6ed8d18b20 100644 --- a/pkg/domain/infra/tunnel/images.go +++ b/pkg/domain/infra/tunnel/images.go @@ -254,6 +254,10 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri options := new(images.PushOptions) options.WithAll(opts.All).WithCompress(opts.Compress).WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithFormat(opts.Format).WithRemoveSignatures(opts.RemoveSignatures).WithQuiet(opts.Quiet).WithCompressionFormat(opts.CompressionFormat).WithProgressWriter(opts.Writer) + if opts.CompressionLevel != nil { + options.WithCompressionLevel(*opts.CompressionLevel) + } + if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined { if s == types.OptionalBoolTrue { options.WithSkipTLSVerify(true) diff --git a/test/e2e/manifest_test.go b/test/e2e/manifest_test.go index 9219b81257..1d777b4be4 100644 --- a/test/e2e/manifest_test.go +++ b/test/e2e/manifest_test.go @@ -319,14 +319,33 @@ var _ = Describe("Podman manifest", func() { )) }) - It("push with compression-format", func() { + It("push with compression-format and compression-level", func() { SkipIfRemote("manifest push to dir not supported in remote mode") - session := podmanTest.Podman([]string{"manifest", "create", "foo"}) + session := podmanTest.Podman([]string{"pull", ALPINE}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - session = podmanTest.Podman([]string{"manifest", "add", "--all", "foo", imageList}) + + dockerfile := `FROM quay.io/libpod/alpine:latest +RUN touch /file +` + podmanTest.BuildImage(dockerfile, "localhost/test", "false") + + session = podmanTest.Podman([]string{"manifest", "create", "foo"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"manifest", "add", "foo", "containers-storage:localhost/test"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) + + // Invalid compression format specified, it must fail + tmpDir := filepath.Join(podmanTest.TempDir, "wrong-compression") + session = podmanTest.Podman([]string{"manifest", "push", "--compression-format", "gzip", "--compression-level", "50", "foo", "oci:" + tmpDir}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(125)) + output := session.ErrorToString() + Expect(output).To(ContainSubstring("invalid compression level")) + dest := filepath.Join(podmanTest.TempDir, "pushed") err := os.MkdirAll(dest, os.ModePerm) Expect(err).ToNot(HaveOccurred()) @@ -434,7 +453,7 @@ var _ = Describe("Podman manifest", func() { Expect(output).To(ContainSubstring("Writing manifest to image destination")) Expect(output).To(ContainSubstring("Storing signatures")) - push = podmanTest.Podman([]string{"manifest", "push", "--tls-verify=false", "--creds=podmantest:wrongpasswd", "foo", "localhost:" + registry.Port + "/credstest"}) + push = podmanTest.Podman([]string{"manifest", "push", "--compression-format=gzip", "--compression-level=2", "--tls-verify=false", "--creds=podmantest:wrongpasswd", "foo", "localhost:" + registry.Port + "/credstest"}) push.WaitWithDefaultTimeout() Expect(push).To(ExitWithError()) Expect(push.ErrorToString()).To(ContainSubstring(": authentication required")) diff --git a/test/e2e/push_test.go b/test/e2e/push_test.go index 72ed8f1d7f..502781a203 100644 --- a/test/e2e/push_test.go +++ b/test/e2e/push_test.go @@ -46,10 +46,18 @@ var _ = Describe("Podman push", func() { Expect(session).Should(Exit(0)) }) - It("podman push to oci with compression-format", func() { + It("podman push to oci with compression-format and compression-level", func() { SkipIfRemote("Remote push does not support dir transport") bbdir := filepath.Join(podmanTest.TempDir, "busybox-oci") - session := podmanTest.Podman([]string{"push", "--compression-format=zstd", "--remove-signatures", ALPINE, + + // Invalid compression format specified, it must fail + session := podmanTest.Podman([]string{"push", "--compression-format=gzip", "--compression-level=40", ALPINE, fmt.Sprintf("oci:%s", bbdir)}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(125)) + output := session.ErrorToString() + Expect(output).To(ContainSubstring("invalid compression level")) + + session = podmanTest.Podman([]string{"push", "--compression-format=zstd", "--remove-signatures", ALPINE, fmt.Sprintf("oci:%s", bbdir)}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) @@ -99,7 +107,7 @@ var _ = Describe("Podman push", func() { Expect(push).Should(Exit(0)) Expect(push.ErrorToString()).To(BeEmpty()) - push = podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:5000/my-alpine"}) + push = podmanTest.Podman([]string{"push", "--compression-format=gzip", "--compression-level=1", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:5000/my-alpine"}) push.WaitWithDefaultTimeout() Expect(push).Should(Exit(0)) output := push.ErrorToString()