From 7ba23cd6fcd5b6c73e4a75fab824fe91ce2ca72a Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Wed, 14 Feb 2024 14:25:10 -0500 Subject: [PATCH] Allow podman pull to specify --retry and --retry-delay Fixes: https://github.com/containers/podman/issues/19770 Signed-off-by: Daniel J Walsh --- cmd/podman/images/pull.go | 26 ++++++++++++++ docs/source/markdown/options/retry-delay.md | 2 +- docs/source/markdown/options/retry.md | 2 +- docs/source/markdown/podman-pull.1.md.in | 8 +++++ pkg/api/handlers/compat/images.go | 21 +++++++++-- pkg/api/handlers/libpod/images_pull.go | 16 +++++++++ pkg/bindings/images/types.go | 4 +++ pkg/bindings/images/types_pull_options.go | 30 ++++++++++++++++ pkg/domain/entities/images.go | 4 +++ pkg/domain/infra/abi/images.go | 10 ++++++ pkg/domain/infra/tunnel/images.go | 6 ++++ test/system/150-login.bats | 40 +++++++++++++++++++++ test/system/helpers.registry.bash | 20 +++++++++++ 13 files changed, 184 insertions(+), 5 deletions(-) diff --git a/cmd/podman/images/pull.go b/cmd/podman/images/pull.go index a1c8b6ca8f..47ab528b6f 100644 --- a/cmd/podman/images/pull.go +++ b/cmd/podman/images/pull.go @@ -113,6 +113,13 @@ func pullFlags(cmd *cobra.Command) { flags.StringArrayVar(&pullOptions.DecryptionKeys, decryptionKeysFlagName, nil, "Key needed to decrypt the image (e.g. /path/to/key.pem)") _ = cmd.RegisterFlagCompletionFunc(decryptionKeysFlagName, completion.AutocompleteDefault) + retryFlagName := "retry" + flags.Uint(retryFlagName, cli.MaxPullPushRetries, "number of times to retry in case of failure when performing pull") + _ = cmd.RegisterFlagCompletionFunc(retryFlagName, completion.AutocompleteNone) + retryDelayFlagName := "retry-delay" + flags.String(retryDelayFlagName, cli.PullPushRetryDelay.String(), "delay between retries in case of pull failures") + _ = cmd.RegisterFlagCompletionFunc(retryDelayFlagName, completion.AutocompleteNone) + if registry.IsRemote() { _ = flags.MarkHidden(decryptionKeysFlagName) } @@ -136,6 +143,25 @@ func imagePull(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("tls-verify") { pullOptions.SkipTLSVerify = types.NewOptionalBool(!pullOptions.TLSVerifyCLI) } + + if cmd.Flags().Changed("retry") { + retry, err := cmd.Flags().GetUint("retry") + if err != nil { + return err + } + + pullOptions.Retry = &retry + } + + if cmd.Flags().Changed("retry-delay") { + val, err := cmd.Flags().GetString("retry-delay") + if err != nil { + return err + } + + pullOptions.RetryDelay = val + } + if cmd.Flags().Changed("authfile") { if err := auth.CheckAuthFile(pullOptions.Authfile); err != nil { return err diff --git a/docs/source/markdown/options/retry-delay.md b/docs/source/markdown/options/retry-delay.md index 8271f4d6a8..bb20a06027 100644 --- a/docs/source/markdown/options/retry-delay.md +++ b/docs/source/markdown/options/retry-delay.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman build, farm build +####> podman build, farm build, pull ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--retry-delay**=*duration* diff --git a/docs/source/markdown/options/retry.md b/docs/source/markdown/options/retry.md index fd13b6e4e1..e3c72b65d8 100644 --- a/docs/source/markdown/options/retry.md +++ b/docs/source/markdown/options/retry.md @@ -1,5 +1,5 @@ ####> This option file is used in: -####> podman build, farm build +####> podman build, farm build, pull ####> If file is edited, make sure the changes ####> are applicable to all of those. #### **--retry**=*attempts* diff --git a/docs/source/markdown/podman-pull.1.md.in b/docs/source/markdown/podman-pull.1.md.in index 0081e00e6a..2046cbe9b0 100644 --- a/docs/source/markdown/podman-pull.1.md.in +++ b/docs/source/markdown/podman-pull.1.md.in @@ -73,6 +73,10 @@ Print the usage statement. Suppress output information when pulling images +@@option retry + +@@option retry-delay + @@option tls-verify @@option variant.container @@ -205,6 +209,10 @@ Storing signatures 3cba58dad5d9b35e755b48b634acb3fdd185ab1c996ac11510cc72c17780e13c ``` +Pull an image with up to 6 retries, delaying 10 seconds between retries in quet mode. +$ podman --remote pull -q --retry 6 --retry-delay 10s ubi9 +4d6addf62a90e392ff6d3f470259eb5667eab5b9a8e03d20b41d0ab910f92170 + ## SEE ALSO **[podman(1)](podman.1.md)**, **[podman-push(1)](podman-push.1.md)**, **[podman-login(1)](podman-login.1.md)**, **[containers-certs.d(5)](https://github.com/containers/image/blob/main/docs/containers-certs.d.5.md)**, **[containers-registries.conf(5)](https://github.com/containers/image/blob/main/docs/containers-registries.conf.5.md)**, **[containers-transports(5)](https://github.com/containers/image/blob/main/docs/containers-transports.5.md)** diff --git a/pkg/api/handlers/compat/images.go b/pkg/api/handlers/compat/images.go index 89d904f183..238277a7d5 100644 --- a/pkg/api/handlers/compat/images.go +++ b/pkg/api/handlers/compat/images.go @@ -257,9 +257,11 @@ func CreateImageFromImage(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) query := struct { - FromImage string `schema:"fromImage"` - Tag string `schema:"tag"` - Platform string `schema:"platform"` + FromImage string `schema:"fromImage"` + Tag string `schema:"tag"` + Platform string `schema:"platform"` + Retry uint `schema:"retry"` + RetryDelay string `schema:"retryDelay"` }{ // This is where you can override the golang default value for one of fields } @@ -290,6 +292,19 @@ func CreateImageFromImage(w http.ResponseWriter, r *http.Request) { } pullOptions.Writer = os.Stderr // allows for debugging on the server + if _, found := r.URL.Query()["retry"]; found { + pullOptions.MaxRetries = &query.Retry + } + + if _, found := r.URL.Query()["retryDelay"]; found { + duration, err := time.ParseDuration(query.RetryDelay) + if err != nil { + utils.Error(w, http.StatusBadRequest, err) + return + } + pullOptions.RetryDelay = &duration + } + // Handle the platform. platformSpecs := strings.Split(query.Platform, "/") pullOptions.OS = platformSpecs[0] // may be empty diff --git a/pkg/api/handlers/libpod/images_pull.go b/pkg/api/handlers/libpod/images_pull.go index 5181a58a76..7075b98b64 100644 --- a/pkg/api/handlers/libpod/images_pull.go +++ b/pkg/api/handlers/libpod/images_pull.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "time" "github.com/containers/common/libimage" "github.com/containers/common/pkg/config" @@ -33,6 +34,8 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) { PullPolicy string `schema:"policy"` Quiet bool `schema:"quiet"` Reference string `schema:"reference"` + Retry uint `schema:"retry"` + RetryDelay string `schema:"retrydelay"` TLSVerify bool `schema:"tlsVerify"` // Platform fields below: Arch string `schema:"Arch"` @@ -95,6 +98,19 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) { return } + if _, found := r.URL.Query()["retry"]; found { + pullOptions.MaxRetries = &query.Retry + } + + if _, found := r.URL.Query()["retrydelay"]; found { + duration, err := time.ParseDuration(query.RetryDelay) + if err != nil { + utils.Error(w, http.StatusBadRequest, err) + return + } + pullOptions.RetryDelay = &duration + } + // Let's keep thing simple when running in quiet mode and pull directly. if query.Quiet { images, err := runtime.LibimageRuntime().Pull(r.Context(), query.Reference, pullPolicy, pullOptions) diff --git a/pkg/bindings/images/types.go b/pkg/bindings/images/types.go index 25595adc4c..6c4cb07dbd 100644 --- a/pkg/bindings/images/types.go +++ b/pkg/bindings/images/types.go @@ -217,6 +217,10 @@ type PullOptions struct { // Quiet can be specified to suppress pull progress when pulling. Ignored // for remote calls. Quiet *bool + // Retry number of times to retry pull in case of failure + Retry *uint + // RetryDelay between retries in case of pull failures + RetryDelay *string // SkipTLSVerify to skip HTTPS and certificate verification. SkipTLSVerify *bool `schema:"-"` // Username for authenticating against the registry. diff --git a/pkg/bindings/images/types_pull_options.go b/pkg/bindings/images/types_pull_options.go index cf48878e2e..7a10a5c391 100644 --- a/pkg/bindings/images/types_pull_options.go +++ b/pkg/bindings/images/types_pull_options.go @@ -138,6 +138,36 @@ func (o *PullOptions) GetQuiet() bool { return *o.Quiet } +// WithRetry set field Retry to given value +func (o *PullOptions) WithRetry(value uint) *PullOptions { + o.Retry = &value + return o +} + +// GetRetry returns value of field Retry +func (o *PullOptions) GetRetry() uint { + if o.Retry == nil { + var z uint + return z + } + return *o.Retry +} + +// WithRetryDelay set field RetryDelay to given value +func (o *PullOptions) WithRetryDelay(value string) *PullOptions { + o.RetryDelay = &value + return o +} + +// GetRetryDelay returns value of field RetryDelay +func (o *PullOptions) GetRetryDelay() string { + if o.RetryDelay == nil { + var z string + return z + } + return *o.RetryDelay +} + // WithSkipTLSVerify set field SkipTLSVerify to given value func (o *PullOptions) WithSkipTLSVerify(value bool) *PullOptions { o.SkipTLSVerify = &value diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index 247e9cf643..96ed74617e 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -102,6 +102,10 @@ type ImagePullOptions struct { // Quiet can be specified to suppress pull progress when pulling. Ignored // for remote calls. Quiet bool + // Retry number of times to retry pull in case of failure + Retry *uint + // RetryDelay between retries in case of pull failures + RetryDelay string // SignaturePolicy to use when pulling. Ignored for remote calls. SignaturePolicy string // SkipTLSVerify to skip HTTPS and certificate verification. diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index f0eb556bc0..be574f090c 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "syscall" + "time" bdefine "github.com/containers/buildah/define" "github.com/containers/common/libimage" @@ -254,6 +255,15 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, options entiti pullOptions.Writer = options.Writer pullOptions.OciDecryptConfig = options.OciDecryptConfig + pullOptions.MaxRetries = options.Retry + if options.RetryDelay != "" { + duration, err := time.ParseDuration(options.RetryDelay) + if err != nil { + return nil, err + } + pullOptions.RetryDelay = &duration + } + if !options.Quiet && pullOptions.Writer == nil { pullOptions.Writer = os.Stderr } diff --git a/pkg/domain/infra/tunnel/images.go b/pkg/domain/infra/tunnel/images.go index d92afc19ac..9090a2b6db 100644 --- a/pkg/domain/infra/tunnel/images.go +++ b/pkg/domain/infra/tunnel/images.go @@ -123,6 +123,12 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, opts entities. options.WithSkipTLSVerify(false) } } + if opts.Retry != nil { + options.WithRetry(*opts.Retry) + } + if opts.RetryDelay != "" { + options.WithRetryDelay(opts.RetryDelay) + } pulledImages, err := images.Pull(ir.ClientCtx, rawImage, options) if err != nil { return nil, err diff --git a/test/system/150-login.bats b/test/system/150-login.bats index f8c2994d2c..0a59371736 100644 --- a/test/system/150-login.bats +++ b/test/system/150-login.bats @@ -327,6 +327,46 @@ function _test_skopeo_credential_sharing() { } +@test "podman images with retry" { + run_podman pull -q --retry 4 --retry-delay "10s" $IMAGE + run_podman 125 pull -q --retry 4 --retry-delay "bogus" $IMAGE + is "$output" 'Error: time: invalid duration "bogus"' "bad retry-delay" + + skip_if_remote "running a local registry doesn't work with podman-remote" + start_registry + authfile=${PODMAN_LOGIN_WORKDIR}/auth-$(random_string 10).json + run_podman login --tls-verify=false \ + --username ${PODMAN_LOGIN_USER} \ + --password-stdin \ + --authfile=$authfile \ + localhost:${PODMAN_LOGIN_REGISTRY_PORT} <<<"${PODMAN_LOGIN_PASS}" + is "$output" "Login Succeeded!" "output from podman login" + + image1="localhost:${PODMAN_LOGIN_REGISTRY_PORT}/test:1.0" + + run_podman tag $IMAGE $image1 + run_podman push --authfile=$authfile \ + --tls-verify=false $mid \ + $image1 + run_podman rmi $image1 + run_podman pull -q --retry 4 --retry-delay "0s" --authfile=$authfile \ + --tls-verify=false $image1 + assert "${output:0:12}" = "$PODMAN_TEST_IMAGE_ID" "First pull (before stopping registry)" + run_podman rmi $image1 + + # This actually STOPs the registry, so the port is unbound... + pause_registry + # ...then, in eight seconds, we start it again + (sleep 8; unpause_registry) & + run_podman 0+w pull -q --retry 4 --retry-delay "5s" --authfile=$authfile \ + --tls-verify=false $image1 + assert "$output" =~ "Failed, retrying in 5s.*Error: initializing.* connection refused" + assert "${lines[-1]:0:12}" = "$PODMAN_TEST_IMAGE_ID" "push should succeed via retry" + unpause_registry + + run_podman rmi $image1 +} + # END cooperation with skopeo # END actual tests ############################################################################### diff --git a/test/system/helpers.registry.bash b/test/system/helpers.registry.bash index 11bb02e252..b6986e64cd 100644 --- a/test/system/helpers.registry.bash +++ b/test/system/helpers.registry.bash @@ -115,3 +115,23 @@ function stop_registry() { die "Socket still seems open" fi } + +function pause_registry() { + if [[ ! -d "$PODMAN_LOGIN_WORKDIR/auth" ]]; then + # No registry running + return + fi + + opts="--storage-driver vfs $(podman_isolation_opts ${PODMAN_LOGIN_WORKDIR})" + run_podman $opts stop registry +} + +function unpause_registry() { + if [[ ! -d "$PODMAN_LOGIN_WORKDIR/auth" ]]; then + # No registry running + return + fi + + opts="--storage-driver vfs $(podman_isolation_opts ${PODMAN_LOGIN_WORKDIR})" + run_podman $opts start registry +}