Merge pull request #13900 from miminar/prune-external-images

Merged by openshift-bot

pkg/cmd/admin/prune/images.go

@@ -38,18 +38,18 @@ var (
 		Remove image stream tags, images, and image layers by age or usage
 
 		This command removes historical image stream tags, unused images, and unreferenced image
-		layers from the integrated registry. It prefers images that have been directly pushed to
-		the registry, but you may specify --all to include images that were imported (if registry
-		mirroring is enabled).
+		layers from the integrated registry. By default, all images are considered as candidates.
+		The command can be instructed to consider only images that have been directly pushed to the
+		registry by supplying --all=false flag.
 
 		By default, the prune operation performs a dry run making no changes to internal registry. A
 		--confirm flag is needed for changes to be effective.
 
-		Only a user with a cluster role %s or higher who is logged-in will be able to actually delete the
-		images.`)
+		Only a user with a cluster role %s or higher who is logged-in will be able to actually
+		delete the images.`)
 
 	imagesExample = templates.Examples(`
-		# See, what the prune command would delete if only images more than an hour old and obsoleted
+	  # See, what the prune command would delete if only images more than an hour old and obsoleted
 	  # by 3 newer revisions under the same tag were considered.
 	  %[1]s %[2]s --keep-tag-revisions=3 --keep-younger-than=60m
 
@@ -89,7 +89,7 @@ type PruneImagesOptions struct {
 
 // NewCmdPruneImages implements the OpenShift cli prune images command.
 func NewCmdPruneImages(f *clientcmd.Factory, parentName, name string, out io.Writer) *cobra.Command {
-	allImages := false
+	allImages := true
 	opts := &PruneImagesOptions{
 		Confirm:            false,
 		KeepYoungerThan:    &defaultKeepYoungerThan,
@@ -113,7 +113,7 @@ func NewCmdPruneImages(f *clientcmd.Factory, parentName, name string, out io.Wri
 	}
 
 	cmd.Flags().BoolVar(&opts.Confirm, "confirm", opts.Confirm, "If true, specify that image pruning should proceed. Defaults to false, displaying what would be deleted but not actually deleting anything.")
-	cmd.Flags().BoolVar(opts.AllImages, "all", *opts.AllImages, "Include images that were not pushed to the registry but have been mirrored by pullthrough. Requires --registry-url")
+	cmd.Flags().BoolVar(opts.AllImages, "all", *opts.AllImages, "Include images that were not pushed to the registry but have been mirrored by pullthrough.")
 	cmd.Flags().DurationVar(opts.KeepYoungerThan, "keep-younger-than", *opts.KeepYoungerThan, "Specify the minimum age of an image for it to be considered a candidate for pruning.")
 	cmd.Flags().IntVar(opts.KeepTagRevisions, "keep-tag-revisions", *opts.KeepTagRevisions, "Specify the number of image revisions for a tag in an image stream that will be preserved.")
 	cmd.Flags().BoolVar(opts.PruneOverSizeLimit, "prune-over-size-limit", *opts.PruneOverSizeLimit, "Specify if images which are exceeding LimitRanges (see 'openshift.io/Image'), specified in the same namespace, should be considered for pruning. This flag cannot be combined with --keep-younger-than nor --keep-tag-revisions.")
@@ -140,11 +140,7 @@ func (o *PruneImagesOptions) Complete(f *clientcmd.Factory, cmd *cobra.Command,
 			o.KeepTagRevisions = nil
 		}
 	}
-	if *o.AllImages {
-		if len(o.RegistryUrlOverride) == 0 {
-			return kcmdutil.UsageError(cmd, "--registry-url must be specified when --all is true")
-		}
-	}
+
 	o.Namespace = metav1.NamespaceAll
 	if cmd.Flags().Lookup("namespace").Changed {
 		var err error

pkg/image/prune/prune.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"sort"
 	"time"
 
 	"github.com/docker/distribution/manifest/schema2"
@@ -267,6 +268,7 @@ func NewPruner(options PrunerOptions) Pruner {
 	if options.PruneOverSizeLimit != nil {
 		algorithm.pruneOverSizeLimit = *options.PruneOverSizeLimit
 	}
+	algorithm.allImages = true
 	if options.AllImages != nil {
 		algorithm.allImages = *options.AllImages
 	}
@@ -635,6 +637,17 @@ func getImageNodes(nodes []gonum.Node) []*imagegraph.ImageNode {
 	return ret
 }
 
+// getImageStreamNodes returns only nodes of type ImageStreamNode.
+func getImageStreamNodes(nodes []gonum.Node) []*imagegraph.ImageStreamNode {
+	ret := []*imagegraph.ImageStreamNode{}
+	for i := range nodes {
+		if node, ok := nodes[i].(*imagegraph.ImageStreamNode); ok {
+			ret = append(ret, node)
+		}
+	}
+	return ret
+}
+
 // edgeKind returns true if the edge from "from" to "to" is of the desired kind.
 func edgeKind(g graph.Graph, from, to gonum.Node, desiredKind string) bool {
 	edge := g.Edge(from, to)
@@ -787,14 +800,60 @@ func pruneImages(g graph.Graph, imageNodes []*imagegraph.ImageNode, imagePruner
 	return errs
 }
 
-func (p *pruner) determineRegistry(imageNodes []*imagegraph.ImageNode) (string, error) {
+// order younger images before older
+type imgByAge []*imageapi.Image
+
+func (ba imgByAge) Len() int      { return len(ba) }
+func (ba imgByAge) Swap(i, j int) { ba[i], ba[j] = ba[j], ba[i] }
+func (ba imgByAge) Less(i, j int) bool {
+	return ba[i].CreationTimestamp.After(ba[j].CreationTimestamp.Time)
+}
+
+// order younger image stream before older
+type isByAge []*imagegraph.ImageStreamNode
+
+func (ba isByAge) Len() int      { return len(ba) }
+func (ba isByAge) Swap(i, j int) { ba[i], ba[j] = ba[j], ba[i] }
+func (ba isByAge) Less(i, j int) bool {
+	return ba[i].ImageStream.CreationTimestamp.After(ba[j].ImageStream.CreationTimestamp.Time)
+}
+
+func (p *pruner) determineRegistry(imageNodes []*imagegraph.ImageNode, isNodes []*imagegraph.ImageStreamNode) (string, error) {
 	if len(p.registryURL) > 0 {
 		return p.registryURL, nil
 	}
 
-	// we only support a single internal registry, and all images have the same registry
-	// so we just take the 1st one and use it
-	pullSpec := imageNodes[0].Image.DockerImageReference
+	var pullSpec string
+	var managedImages []*imageapi.Image
+
+	// 1st try to determine registry url from a pull spec of the youngest managed image
+	for _, node := range imageNodes {
+		if node.Image.Annotations[imageapi.ManagedByOpenShiftAnnotation] != "true" {
+			continue
+		}
+		managedImages = append(managedImages, node.Image)
+	}
+	// be sure to pick up the newest managed image which should have an up to date information
+	sort.Sort(imgByAge(managedImages))
+
+	if len(managedImages) > 0 {
+		pullSpec = managedImages[0].DockerImageReference
+	} else {
+		// 2nd try to get the pull spec from any image stream
+		// Sorting by creation timestamp may not get us up to date info. Modification time would be much
+		// better if there were such an attribute.
+		sort.Sort(isByAge(isNodes))
+		for _, node := range isNodes {
+			if len(node.ImageStream.Status.DockerImageRepository) == 0 {
+				continue
+			}
+			pullSpec = node.ImageStream.Status.DockerImageRepository
+		}
+	}
+
+	if len(pullSpec) == 0 {
+		return "", fmt.Errorf("no managed image found")
+	}
 
 	ref, err := imageapi.ParseDockerImageReference(pullSpec)
 	if err != nil {
@@ -825,7 +884,7 @@ func (p *pruner) Prune(
 		return nil
 	}
 
-	registryURL, err := p.determineRegistry(imageNodes)
+	registryURL, err := p.determineRegistry(imageNodes, getImageStreamNodes(allNodes))
 	if err != nil {
 		return fmt.Errorf("unable to determine registry: %v", err)
 	}

test/extended/images/helper.go

@@ -6,15 +6,19 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path"
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/docker/distribution/digest"
 	dockerclient "github.com/fsouza/go-dockerclient"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/openshift/origin/pkg/client"
 	"github.com/openshift/origin/pkg/image/api"
@@ -26,6 +30,8 @@ const (
 	// There are coefficients used to multiply layer data size to get a rough size of uploaded blob.
 	layerSizeMultiplierForDocker18     = 2.0
 	layerSizeMultiplierForLatestDocker = 0.8
+	digestSHA256GzippedEmptyTar        = digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")
+	digestSha256EmptyTar               = digest.Digest("sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
 )
 
 var (
@@ -220,7 +226,6 @@ func BuildAndPushImageOfSizeWithDocker(
 			return "", fmt.Errorf("Got unexpected push error: %v", err)
 		}
 		if len(imageDigest) == 0 {
-			outSink.Write([]byte("matching digest string\n"))
 			match := rePushedImageDigest.FindStringSubmatch(out)
 			if len(match) < 2 {
 				return imageDigest, fmt.Errorf("Failed to parse digest")
@@ -327,3 +332,94 @@ func calculateRoughDataSize(logger io.Writer, wantedImageSize uint64, numberOfLa
 	// running Docker daemon < 1.9
 	return uint64(float64(wantedImageSize) / (float64(numberOfLayers) * layerSizeMultiplierForDocker18))
 }
+
+// MirrorBlobInRegistry forces a blob of external image to be mirrored in the registry. The function expects
+// the blob not to exist before a GET request is issued. The function blocks until the blob is mirrored or the
+// given timeout passes.
+func MirrorBlobInRegistry(oc *exutil.CLI, dgst digest.Digest, repository string, timeout time.Duration) error {
+	presentGlobally, inRepository, err := IsBlobStoredInRegistry(oc, dgst, repository)
+	if err != nil {
+		return err
+	}
+	if presentGlobally || inRepository {
+		return fmt.Errorf("blob %q is already present in the registry", dgst.String())
+	}
+	registryURL, err := GetDockerRegistryURL(oc)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/v2/%s/blobs/%s", registryURL, repository, dgst.String()), nil)
+	if err != nil {
+		return err
+	}
+	token, err := oc.Run("whoami").Args("-t").Output()
+	if err != nil {
+		return err
+	}
+	req.Header.Set("range", "bytes=0-1")
+	req.Header.Set("Authorization", "Bearer "+token)
+	c := http.Client{}
+	resp, err := c.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("expected status %d for request '%s %s', got %d", http.StatusOK, req.Method, req.URL.String(), resp.StatusCode)
+	}
+
+	return wait.Poll(time.Second, timeout, func() (bool, error) {
+		globally, inRepo, err := IsBlobStoredInRegistry(oc, dgst, repository)
+		return globally || inRepo, err
+	})
+}
+
+// IsEmptyDigest returns true if the given digest matches one of empty blobs.
+func IsEmptyDigest(dgst digest.Digest) bool {
+	return dgst == digestSha256EmptyTar || dgst == digestSHA256GzippedEmptyTar
+}
+
+func pathExistsInRegistry(oc *exutil.CLI, pthComponents ...string) (bool, error) {
+	pth := path.Join(append([]string{"/registry/docker/registry/v2"}, pthComponents...)...)
+	cmd := fmt.Sprintf("[ -e %s ] && echo exists || echo missing", pth)
+	out, err := oc.SetNamespace(metav1.NamespaceDefault).AsAdmin().Run("rsh").Args(
+		"dc/docker-registry", "/bin/sh", "-c", cmd).Output()
+	if err != nil {
+		return false, fmt.Errorf("failed to check for blob existence: %v", err)
+	}
+	return strings.HasPrefix(out, "exists"), nil
+}
+
+// IsBlobStoredInRegistry verifies a presence of the given blob on registry's storage. The registry must be
+// deployed with a filesystem storage driver. If repository is given, the presence will be verified also for
+// layer link inside the ${repository}/_layers directory. First returned bool says whether the blob is present
+// globally in the registry's storage. The second says whether the blob is linked in the given repository.
+func IsBlobStoredInRegistry(
+	oc *exutil.CLI,
+	dgst digest.Digest,
+	repository string,
+) (bool, bool, error) {
+	present, err := pathExistsInRegistry(
+		oc,
+		"blobs",
+		string(dgst.Algorithm()),
+		dgst.Hex()[0:2],
+		dgst.Hex(),
+		"data")
+	if err != nil || !present {
+		return false, false, err
+	}
+
+	presentInRepository := false
+	if len(repository) > 0 {
+		presentInRepository, err = pathExistsInRegistry(oc,
+			"repositories",
+			repository,
+			"_layers",
+			string(dgst.Algorithm()),
+			dgst.Hex(),
+			"link")
+	}
+	return present, presentInRepository, err
+}