create: support images with invalid platform

Much to my regret, there is a number of images in the wild with invalid
platforms breaking the platform checks in libimage that want to make
sure that a local image is matching the expected platform.

Imagine a `podman run --arch=arm64 fedora` with a local amd64 fedora
image.  We really shouldn't use the local one in this case and pull down
the arm64 one.

The strict platform checks in libimage in combination with invalid
platforms in images surfaced in Podman being able to pull an image but
failing to look it up in subsequent presence checks.  A `podman run`
would hence pull such an image but fail to create the container.

Support images with invalid platforms by vendoring the latest HEAD from
containers/common.  Also remove the partially implemented pull-policy
logic from Podman and let libimage handle that entirely.  However,
whenever --arch, --os or --platform are specified, the pull policy will
be forced to "newer".  This way, we pessimistically assume that the
local image has an invalid platform and we reach out to the registry.
If there's a newer image (i.e., one with a different digest), we'll pull
it down.

Please note that most of the logic has either already been implemented
in libimage or been moved down which allows for removing some clutter
from Podman.

[NO TESTS NEEDED] since c/common has new tests.  Podman can rely on the
existing tests.

Fixes: #10648
Fixes: #10682
Signed-off-by: Valentin Rothberg <[email protected]>

cmd/podman/containers/create.go

@@ -8,17 +8,14 @@ import (
 	"strings"
 
 	"github.com/containers/common/pkg/config"
-	storageTransport "github.com/containers/image/v5/storage"
 	"github.com/containers/image/v5/transports/alltransports"
 	"github.com/containers/podman/v3/cmd/podman/common"
 	"github.com/containers/podman/v3/cmd/podman/registry"
 	"github.com/containers/podman/v3/cmd/podman/utils"
 	"github.com/containers/podman/v3/pkg/domain/entities"
 	"github.com/containers/podman/v3/pkg/specgen"
 	"github.com/containers/podman/v3/pkg/util"
-	"github.com/containers/storage"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -236,30 +233,12 @@ func createInit(c *cobra.Command) error {
 	return nil
 }
 
-// TODO: we should let the backend take care of the pull policy (which it
-// does!). The code below is at risk of causing regression and code divergence.
 func pullImage(imageName string) (string, error) {
 	pullPolicy, err := config.ValidatePullPolicy(cliVals.Pull)
 	if err != nil {
 		return "", err
 	}
 
-	// Check if the image is missing and hence if we need to pull it.
-	imageMissing := true
-	imageRef, err := alltransports.ParseImageName(imageName)
-	switch {
-	case err != nil:
-		// Assume we specified a local image without the explicit storage transport.
-		fallthrough
-
-	case imageRef.Transport().Name() == storageTransport.Transport.Name():
-		br, err := registry.ImageEngine().Exists(registry.GetContext(), imageName)
-		if err != nil {
-			return "", err
-		}
-		imageMissing = !br.Value
-	}
-
 	if cliVals.Platform != "" || cliVals.Arch != "" || cliVals.OS != "" {
 		if cliVals.Platform != "" {
 			if cliVals.Arch != "" || cliVals.OS != "" {
@@ -271,31 +250,28 @@ func pullImage(imageName string) (string, error) {
 				cliVals.Arch = split[1]
 			}
 		}
+	}
 
-		if pullPolicy != config.PullPolicyAlways {
-			logrus.Info("--platform --arch and --os causes the pull policy to be \"always\"")
-			pullPolicy = config.PullPolicyAlways
-		}
+	pullReport, pullErr := registry.ImageEngine().Pull(registry.GetContext(), imageName, entities.ImagePullOptions{
+		Authfile:        cliVals.Authfile,
+		Quiet:           cliVals.Quiet,
+		Arch:            cliVals.Arch,
+		OS:              cliVals.OS,
+		Variant:         cliVals.Variant,
+		SignaturePolicy: cliVals.SignaturePolicy,
+		PullPolicy:      pullPolicy,
+	})
+	if pullErr != nil {
+		return "", pullErr
 	}
 
-	if imageMissing || pullPolicy == config.PullPolicyAlways {
-		if pullPolicy == config.PullPolicyNever {
-			return "", errors.Wrap(storage.ErrImageUnknown, imageName)
-		}
-		pullReport, pullErr := registry.ImageEngine().Pull(registry.GetContext(), imageName, entities.ImagePullOptions{
-			Authfile:        cliVals.Authfile,
-			Quiet:           cliVals.Quiet,
-			Arch:            cliVals.Arch,
-			OS:              cliVals.OS,
-			Variant:         cliVals.Variant,
-			SignaturePolicy: cliVals.SignaturePolicy,
-			PullPolicy:      pullPolicy,
-		})
-		if pullErr != nil {
-			return "", pullErr
-		}
+	// Return the input name such that the image resolves to correct
+	// repo/tag in the backend (see #8082).  Unless we're referring to
+	// the image via a transport.
+	if _, err := alltransports.ParseImageName(imageName); err == nil {
 		imageName = pullReport.Images[0]
 	}
+
 	return imageName, nil
 }
 

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/containernetworking/cni v0.8.1
 	github.com/containernetworking/plugins v0.9.1
 	github.com/containers/buildah v1.21.1
-	github.com/containers/common v0.40.1-0.20210617134614-c6578d76fb0d
+	github.com/containers/common v0.40.2-0.20210623133759-d13a31743aec
 	github.com/containers/conmon v2.0.20+incompatible
 	github.com/containers/image/v5 v5.13.2
 	github.com/containers/ocicrypt v1.1.1

go.sum

@@ -221,12 +221,11 @@ github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRD
 github.com/containers/buildah v1.21.1 h1:e9LmTCUKUBLg72v5DnIOT/wc8ffkfB7LbpQBywLZo20=
 github.com/containers/buildah v1.21.1/go.mod h1:yPdlpVd93T+i91yGxrJbW1YOWrqN64j5ZhHOZmHUejs=
 github.com/containers/common v0.38.4/go.mod h1:egfpX/Y3+19Dz4Wa1eRZDdgzoEOeneieF9CQppKzLBg=
-github.com/containers/common v0.40.1-0.20210617134614-c6578d76fb0d h1:PaS/t2XcyxEDOr685T+3HPMyMqN99UPcj6I92nqIDH8=
-github.com/containers/common v0.40.1-0.20210617134614-c6578d76fb0d/go.mod h1:+zxauZzkurY5tbQGDxrCV6rF694RX1olXyYRVJHrzWo=
+github.com/containers/common v0.40.2-0.20210623133759-d13a31743aec h1:ZcteA2klZSZAZgVonwJAqezF6hdO9SMKUy49ZHXZd38=
+github.com/containers/common v0.40.2-0.20210623133759-d13a31743aec/go.mod h1:J23CfuhN1fAg85q5HxS6SKYhKbGqmqieKQqoHaQbEI8=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
 github.com/containers/image/v5 v5.12.0/go.mod h1:VasTuHmOw+uD0oHCfApQcMO2+36SfyncoSahU7513Xs=
-github.com/containers/image/v5 v5.13.2-0.20210617132750-db0df5e0cf5e/go.mod h1:GkWursKDlDcUIT7L7vZf70tADvZCk/Ga0wgS0MuF0ag=
 github.com/containers/image/v5 v5.13.2 h1:AgYunV/9d2fRkrmo23wH2MkqeHolFd6oQCkK+1PpuFA=
 github.com/containers/image/v5 v5.13.2/go.mod h1:GkWursKDlDcUIT7L7vZf70tADvZCk/Ga0wgS0MuF0ag=
 github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE=

pkg/api/handlers/libpod/images_pull.go

@@ -26,14 +26,16 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) {
 	runtime := r.Context().Value("runtime").(*libpod.Runtime)
 	decoder := r.Context().Value("decoder").(*schema.Decoder)
 	query := struct {
-		Reference string `schema:"reference"`
-		OS        string `schema:"OS"`
-		Arch      string `schema:"Arch"`
-		Variant   string `schema:"Variant"`
-		TLSVerify bool   `schema:"tlsVerify"`
-		AllTags   bool   `schema:"allTags"`
+		Reference  string `schema:"reference"`
+		OS         string `schema:"OS"`
+		Arch       string `schema:"Arch"`
+		Variant    string `schema:"Variant"`
+		TLSVerify  bool   `schema:"tlsVerify"`
+		AllTags    bool   `schema:"allTags"`
+		PullPolicy string `schema:"policy"`
 	}{
-		TLSVerify: true,
+		TLSVerify:  true,
+		PullPolicy: "always",
 	}
 
 	if err := decoder.Decode(&query, r.URL.Query()); err != nil {
@@ -83,12 +85,18 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) {
 
 	pullOptions.Writer = writer
 
+	pullPolicy, err := config.ParsePullPolicy(query.PullPolicy)
+	if err != nil {
+		utils.Error(w, "failed to parse pull policy", http.StatusBadRequest, err)
+		return
+	}
+
 	var pulledImages []*libimage.Image
 	var pullError error
 	runCtx, cancel := context.WithCancel(r.Context())
 	go func() {
 		defer cancel()
-		pulledImages, pullError = runtime.LibimageRuntime().Pull(runCtx, query.Reference, config.PullPolicyAlways, pullOptions)
+		pulledImages, pullError = runtime.LibimageRuntime().Pull(runCtx, query.Reference, pullPolicy, pullOptions)
 	}()
 
 	flush := func() {

pkg/api/server/register_images.go

@@ -974,6 +974,10 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error {
 	//     description: Pull image for the specified variant.
 	//     type: string
 	//   - in: query
+	//     name: policy
+	//     description: Pull policy, "always" (default), "missing", "newer", "never".
+	//     type: string
+	//   - in: query
 	//     name: tlsVerify
 	//     description: Require TLS verification.
 	//     type: boolean

pkg/bindings/images/pull.go

@@ -13,7 +13,7 @@ import (
 	"github.com/containers/podman/v3/pkg/auth"
 	"github.com/containers/podman/v3/pkg/bindings"
 	"github.com/containers/podman/v3/pkg/domain/entities"
-	"github.com/hashicorp/go-multierror"
+	"github.com/containers/podman/v3/pkg/errorhandling"
 	"github.com/pkg/errors"
 )
 
@@ -65,7 +65,7 @@ func Pull(ctx context.Context, rawImage string, options *PullOptions) ([]string,
 
 	dec := json.NewDecoder(response.Body)
 	var images []string
-	var mErr error
+	var pullErrors []error
 	for {
 		var report entities.ImagePullReport
 		if err := dec.Decode(&report); err != nil {
@@ -77,7 +77,7 @@ func Pull(ctx context.Context, rawImage string, options *PullOptions) ([]string,
 
 		select {
 		case <-response.Request.Context().Done():
-			return images, mErr
+			break
 		default:
 			// non-blocking select
 		}
@@ -86,13 +86,13 @@ func Pull(ctx context.Context, rawImage string, options *PullOptions) ([]string,
 		case report.Stream != "":
 			fmt.Fprint(stderr, report.Stream)
 		case report.Error != "":
-			mErr = multierror.Append(mErr, errors.New(report.Error))
+			pullErrors = append(pullErrors, errors.New(report.Error))
 		case len(report.Images) > 0:
 			images = report.Images
 		case report.ID != "":
 		default:
 			return images, errors.Errorf("failed to parse pull results stream, unexpected input: %v", report)
 		}
 	}
-	return images, mErr
+	return images, errorhandling.JoinErrors(pullErrors)
 }

pkg/bindings/images/types.go

@@ -147,6 +147,9 @@ type PullOptions struct {
 	// OS will overwrite the local operating system (OS) for image
 	// pulls.
 	OS *string
+	// Policy is the pull policy. Supported values are "missing", "never",
+	// "newer", "always". An empty string defaults to "always".
+	Policy *string
 	// Password for authenticating against the registry.
 	Password *string
 	// Quiet can be specified to suppress pull progress when pulling.  Ignored

pkg/bindings/images/types_pull_options.go

@@ -84,6 +84,22 @@ func (o *PullOptions) GetOS() string {
 	return *o.OS
 }
 
+// WithPolicy
+func (o *PullOptions) WithPolicy(value string) *PullOptions {
+	v := &value
+	o.Policy = v
+	return o
+}
+
+// GetPolicy
+func (o *PullOptions) GetPolicy() string {
+	var policy string
+	if o.Policy == nil {
+		return policy
+	}
+	return *o.Policy
+}
+
 // WithPassword
 func (o *PullOptions) WithPassword(value string) *PullOptions {
 	v := &value

pkg/domain/infra/tunnel/images.go

@@ -107,7 +107,7 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, opts entities.
 	options := new(images.PullOptions)
 	options.WithAllTags(opts.AllTags).WithAuthfile(opts.Authfile).WithArch(opts.Arch).WithOS(opts.OS)
 	options.WithVariant(opts.Variant).WithPassword(opts.Password)
-	options.WithQuiet(opts.Quiet).WithUsername(opts.Username)
+	options.WithQuiet(opts.Quiet).WithUsername(opts.Username).WithPolicy(opts.PullPolicy.String())
 	if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined {
 		if s == types.OptionalBoolTrue {
 			options.WithSkipTLSVerify(true)

pkg/errorhandling/errorhandling.go

@@ -15,6 +15,12 @@ func JoinErrors(errs []error) error {
 		return nil
 	}
 
+	// If there's just one error, return it.  This prevents the "%d errors
+	// occurred:" header plus list from the multierror package.
+	if len(errs) == 1 {
+		return errs[0]
+	}
+
 	// `multierror` appends new lines which we need to remove to prevent
 	// blank lines when printing the error.
 	var multiE *multierror.Error
@@ -24,9 +30,6 @@ func JoinErrors(errs []error) error {
 	if finalErr == nil {
 		return finalErr
 	}
-	if len(multiE.WrappedErrors()) == 1 && logrus.IsLevelEnabled(logrus.TraceLevel) {
-		return multiE.WrappedErrors()[0]
-	}
 	return errors.New(strings.TrimSpace(finalErr.Error()))
 }
 

pkg/specgen/generate/container.go

@@ -24,7 +24,8 @@ func CompleteSpec(ctx context.Context, r *libpod.Runtime, s *specgen.SpecGenerat
 	var inspectData *libimage.ImageData
 	var err error
 	if s.Image != "" {
-		newImage, _, err = r.LibimageRuntime().LookupImage(s.Image, nil)
+		lookupOptions := &libimage.LookupImageOptions{IgnorePlatform: true}
+		newImage, _, err = r.LibimageRuntime().LookupImage(s.Image, lookupOptions)
 		if err != nil {
 			return nil, err
 		}

pkg/specgen/generate/container_create.go

@@ -92,7 +92,8 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener
 		options = append(options, libpod.WithRootFS(s.Rootfs))
 	} else {
 		var resolvedImageName string
-		newImage, resolvedImageName, err = rt.LibimageRuntime().LookupImage(s.Image, nil)
+		lookupOptions := &libimage.LookupImageOptions{IgnorePlatform: true}
+		newImage, resolvedImageName, err = rt.LibimageRuntime().LookupImage(s.Image, lookupOptions)
 		if err != nil {
 			return nil, err
 		}

test/system/255-auto-update.bats

@@ -194,7 +194,7 @@ function _confirm_update() {
     run_podman 125 auto-update
     update_log=$output
     is "$update_log" ".*invalid auto-update policy.*" "invalid policy setup"
-    is "$update_log" ".*1 error occurred.*" "invalid policy setup"
+    is "$update_log" ".*Error: invalid auto-update policy.*" "invalid policy setup"
 
     local n_updated=$(grep -c 'Trying to pull' <<<"$update_log")
     is "$n_updated" "2" "Number of images updated from registry."