Skip to content

Commit

Permalink
Retry pulling image
Browse files Browse the repository at this point in the history
Wrap the inner helper in the retry function. Functions pullimage failed with retriable error will default maxretry 3 times using exponential backoff.

Signed-off-by: Qi Wang <[email protected]>
  • Loading branch information
QiWang19 committed Aug 4, 2020
1 parent d4cf3c5 commit 42d756d
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 10 deletions.
7 changes: 5 additions & 2 deletions libpod/image/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"syscall"
"time"

"github.com/containers/common/pkg/retry"
cp "github.com/containers/image/v5/copy"
"github.com/containers/image/v5/directory"
dockerarchive "github.com/containers/image/v5/docker/archive"
Expand Down Expand Up @@ -75,6 +76,8 @@ type InfoImage struct {
Layers []LayerInfo
}

const maxRetry = 3

// ImageFilter is a function to determine whether a image is included
// in command output. Images to be outputted are tested using the function.
// A true return will include the image, a false return will exclude it.
Expand Down Expand Up @@ -158,7 +161,7 @@ func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile
if signaturePolicyPath == "" {
signaturePolicyPath = ir.SignaturePolicyPath
}
imageName, err := ir.pullImageFromHeuristicSource(ctx, name, writer, authfile, signaturePolicyPath, signingoptions, dockeroptions, label)
imageName, err := ir.pullImageFromHeuristicSource(ctx, name, writer, authfile, signaturePolicyPath, signingoptions, dockeroptions, &retry.RetryOptions{MaxRetry: maxRetry}, label)
if err != nil {
return nil, errors.Wrapf(err, "unable to pull %s", name)
}
Expand All @@ -176,7 +179,7 @@ func (ir *Runtime) LoadFromArchiveReference(ctx context.Context, srcRef types.Im
if signaturePolicyPath == "" {
signaturePolicyPath = ir.SignaturePolicyPath
}
imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{})
imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{MaxRetry: maxRetry})
if err != nil {
return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef))
}
Expand Down
19 changes: 11 additions & 8 deletions libpod/image/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"path/filepath"
"strings"

"github.com/containers/common/pkg/retry"
cp "github.com/containers/image/v5/copy"
"github.com/containers/image/v5/directory"
"github.com/containers/image/v5/docker"
Expand Down Expand Up @@ -218,7 +219,7 @@ func toLocalImageName(imageName string) string {

// pullImageFromHeuristicSource pulls an image based on inputName, which is heuristically parsed and may involve configured registries.
// Use pullImageFromReference if the source is known precisely.
func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName string, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, label *string) ([]string, error) {
func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName string, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, retryOptions *retry.RetryOptions, label *string) ([]string, error) {
span, _ := opentracing.StartSpanFromContext(ctx, "pullImageFromHeuristicSource")
defer span.Finish()

Expand Down Expand Up @@ -247,11 +248,11 @@ func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName s
return nil, errors.Wrapf(err, "error determining pull goal for image %q", inputName)
}
}
return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, label)
return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, retryOptions, label)
}

// pullImageFromReference pulls an image from a types.imageReference.
func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.ImageReference, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions) ([]string, error) {
func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.ImageReference, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, retryOptions *retry.RetryOptions) ([]string, error) {
span, _ := opentracing.StartSpanFromContext(ctx, "pullImageFromReference")
defer span.Finish()

Expand All @@ -264,7 +265,7 @@ func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.Imag
if err != nil {
return nil, errors.Wrapf(err, "error determining pull goal for image %q", transports.ImageName(srcRef))
}
return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, nil)
return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, retryOptions, nil)
}

func cleanErrorMessage(err error) string {
Expand All @@ -274,7 +275,7 @@ func cleanErrorMessage(err error) string {
}

// doPullImage is an internal helper interpreting pullGoal. Almost everyone should call one of the callers of doPullImage instead.
func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goal pullGoal, writer io.Writer, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, label *string) ([]string, error) {
func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goal pullGoal, writer io.Writer, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, retryOptions *retry.RetryOptions, label *string) ([]string, error) {
span, _ := opentracing.StartSpanFromContext(ctx, "doPullImage")
defer span.Finish()

Expand Down Expand Up @@ -310,9 +311,11 @@ func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goa
return nil, err
}
}

_, err = cp.Image(ctx, policyContext, imageInfo.dstRef, imageInfo.srcRef, copyOptions)
if err != nil {
imageInfo := imageInfo
if err = retry.RetryIfNecessary(ctx, func() error {
_, err = cp.Image(ctx, policyContext, imageInfo.dstRef, imageInfo.srcRef, copyOptions)
return err
}, retryOptions); err != nil {
pullErrors = multierror.Append(pullErrors, err)
logrus.Debugf("Error pulling image ref %s: %v", imageInfo.srcRef.StringWithinTransport(), err)
if writer != nil {
Expand Down
87 changes: 87 additions & 0 deletions vendor/github.com/containers/common/pkg/retry/retry.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions vendor/modules.txt
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ github.com/containers/common/pkg/auth
github.com/containers/common/pkg/capabilities
github.com/containers/common/pkg/cgroupv2
github.com/containers/common/pkg/config
github.com/containers/common/pkg/retry
github.com/containers/common/pkg/sysinfo
github.com/containers/common/version
# github.com/containers/conmon v2.0.19+incompatible
Expand Down

0 comments on commit 42d756d

Please sign in to comment.