Merge pull request #1713 from justaugustus/krel-ci-build

pkg/build: Validate images exist before proceeding with CI builds

images/releng/k8s-ci-builder/Dockerfile

@@ -78,6 +78,10 @@ RUN echo "Installing Packages ..." \
             gnupg2 \
             jq \
             kmod \
+            libassuan-dev \
+            libbtrfs-dev \
+            libdevmapper-dev \
+            libgpgme-dev \
             lsb-release \
             mercurial \
             openssh-client \
@@ -127,6 +131,14 @@ ARG OLD_BAZEL_VERSION
 COPY --from=old-bazel \
     /usr/local/lib/bazel/bin/bazel-real /usr/local/lib/bazel/bin/bazel-${OLD_BAZEL_VERSION}
 
+ARG SKOPEO_VERSION
+RUN git clone https://github.com/containers/skopeo $GOPATH/src/github.com/containers/skopeo
+RUN cd $GOPATH/src/github.com/containers/skopeo \
+    && git checkout ${SKOPEO_VERSION} \
+    && make bin/skopeo \
+    && cp bin/skopeo /usr/local/bin \
+    && rm -rf $GOPATH/src/github.com/containers/skopeo
+
 # Copy in release tools from kubernetes/release
 WORKDIR /
 COPY --from=builder /go/bin/* ./

images/releng/k8s-ci-builder/Makefile

@@ -27,10 +27,14 @@ TAG ?= $(shell git describe --tags --always --dirty)
 GO_VERSION ?= 1.15.3
 BAZEL_VERSION ?= 3.4.1
 OLD_BAZEL_VERSION ?= 2.2.0
+IMAGE_ARG ?= $(IMAGE):$(TAG)-$(CONFIG)
+SKOPEO_VERSION ?= v1.2.0
 
 BUILD_ARGS = --build-arg=GO_VERSION=$(GO_VERSION) \
 							--build-arg=BAZEL_VERSION=$(BAZEL_VERSION) \
-							--build-arg=OLD_BAZEL_VERSION=$(OLD_BAZEL_VERSION)
+							--build-arg=OLD_BAZEL_VERSION=$(OLD_BAZEL_VERSION) \
+							--build-arg=IMAGE_ARG=$(IMAGE_ARG) \
+							--build-arg=SKOPEO_VERSION=$(SKOPEO_VERSION)
 
 # Ensure support for 'docker buildx' and 'docker manifest' commands
 export DOCKER_CLI_EXPERIMENTAL=enabled

images/releng/k8s-ci-builder/cloudbuild.yaml

@@ -24,6 +24,8 @@ steps:
     - GO_VERSION=${_GO_VERSION}
     - BAZEL_VERSION=${_BAZEL_VERSION}
     - OLD_BAZEL_VERSION=${_OLD_BAZEL_VERSION}
+    - IMAGE_ARG='gcr.io/$PROJECT_ID/k8s-ci-builder:${_GIT_TAG}-${_CONFIG}'
+    - SKOPEO_VERSION=${_SKOPEO_VERSION}
     args:
     - '-c'
     - |
@@ -39,6 +41,7 @@ substitutions:
   _GO_VERSION: '0.0.0'
   _BAZEL_VERSION: '0.0.0'
   _OLD_BAZEL_VERSION: '0.0.0'
+  _SKOPEO_VERSION: 'v0.0.0'
 
 tags:
 - 'k8s-ci-builder'

images/releng/k8s-ci-builder/variants.yaml

@@ -4,18 +4,22 @@ variants:
     GO_VERSION: '1.15.3'
     BAZEL_VERSION: '3.4.1'
     OLD_BAZEL_VERSION: '2.2.0'
+    SKOPEO_VERSION: 'v1.2.0'
   '1.19':
     CONFIG: '1.19'
     GO_VERSION: '1.15.3'
     BAZEL_VERSION: '2.2.0'
     OLD_BAZEL_VERSION: '0.23.2'
+    SKOPEO_VERSION: 'v1.2.0'
   '1.18':
     CONFIG: '1.18'
     GO_VERSION: '1.13.15'
     BAZEL_VERSION: '2.2.0'
     OLD_BAZEL_VERSION: '0.23.2'
+    SKOPEO_VERSION: 'v1.2.0'
   '1.17':
     CONFIG: '1.17'
     GO_VERSION: '1.13.15'
     BAZEL_VERSION: '2.2.0'
     OLD_BAZEL_VERSION: '0.23.2'
+    SKOPEO_VERSION: 'v1.2.0'

pkg/build/ci.go

@@ -146,17 +146,23 @@ func (bi *Instance) checkBuildExists() (bool, error) {
 
 	// TODO: Do we need to handle the errors more effectively?
 	existErrors := []error{}
-	for i, path := range gcsBuildPaths {
+	for _, path := range gcsBuildPaths {
 		logrus.Infof("Checking if GCS build path (%s) exists", path)
 		exists, existErr := gcs.PathExists(path)
 		if existErr != nil || !exists {
 			existErrors = append(existErrors, existErr)
 		}
+	}
 
-		if i == len(gcsBuildPaths)-1 && len(existErrors) == 0 {
-			logrus.Infof("Build already exists. Exiting...")
-			return true, nil
-		}
+	images := release.NewImages()
+	imagesExist, imagesExistErr := images.Exists(bi.opts.Registry, version, bi.opts.Fast)
+	if imagesExistErr != nil {
+		existErrors = append(existErrors, imagesExistErr)
+	}
+
+	if imagesExist && len(existErrors) == 0 {
+		logrus.Infof("Build already exists. Exiting...")
+		return true, nil
 	}
 
 	logrus.Infof("The following error(s) occurred while looking for a build: %v", existErrors)

pkg/release/images.go

@@ -79,7 +79,7 @@ func (*defaultCommandClient) RepoTagFromTarball(path string) (string, error) {
 
 var tagRegex = regexp.MustCompile(`^.+/(.+):.+$`)
 
-// PublishImages relases container images to the provided target registry
+// PublishImages releases container images to the provided target registry
 // was in releaselib.sh: release::docker::release
 func (i *Images) Publish(registry, version, buildPath string) error {
 	version = i.normalizeVersion(version)
@@ -239,6 +239,92 @@ func (i *Images) Validate(registry, version, buildPath string) error {
 	return nil
 }
 
+// Exists verifies that a set of image manifests exists on a specified remote
+// registry. This is a simpler check than Validate, which doesn't presuppose the
+// existence of a local build directory. Used in CI builds to quickly validate
+// if a build is actually required.
+func (i *Images) Exists(registry, version string, fast bool) (bool, error) {
+	logrus.Infof("Validating image manifests in %s", registry)
+	version = i.normalizeVersion(version)
+
+	// TODO: Maybe pull this out into a var?
+	manifestImages := []string{
+		"conformance",
+		"kube-apiserver",
+		"kube-controller-manager",
+		"kube-proxy",
+		"kube-scheduler",
+	}
+
+	// TODO: Maybe pull this out into a var?
+	arches := []string{
+		"amd64",
+		"arm",
+		"arm64",
+		"ppc64le",
+		"s390x",
+	}
+
+	// TODO: Maybe pull this out into a var?
+	if fast {
+		arches = []string{
+			"amd64",
+		}
+	}
+
+	for _, image := range manifestImages {
+		imageVersion := fmt.Sprintf("%s/%s:%s", registry, image, version)
+
+		manifest, err := i.client.ExecuteOutput(
+			"skopeo", "inspect", fmt.Sprintf("docker://%s", imageVersion), "--raw",
+		)
+		if err != nil {
+			return false, errors.Wrapf(
+				err, "get remote manifest from %s", imageVersion,
+			)
+		}
+		manifestFile, err := ioutil.TempFile("", "manifest-")
+		if err != nil {
+			return false, errors.Wrap(err, "create temp file for manifest")
+		}
+		if _, err := manifestFile.WriteString(manifest); err != nil {
+			return false, errors.Wrapf(
+				err, "write manifest to %s", manifestFile.Name(),
+			)
+		}
+		defer os.RemoveAll(manifestFile.Name())
+
+		for _, arch := range arches {
+			logrus.Infof(
+				"Checking image digest for %s on %s architecture", image, arch,
+			)
+
+			digest, err := i.client.ExecuteOutput(
+				"jq", "--arg", "a", arch, "-r",
+				".manifests[] | select(.platform.architecture == $a) | .digest",
+				manifestFile.Name(),
+			)
+			if err != nil {
+				return false, errors.Wrapf(
+					err, "get digest from manifest file %s for arch %s",
+					manifestFile.Name(), arch,
+				)
+			}
+
+			if digest == "" {
+				return false, errors.Errorf(
+					"could not find the image digest for %s on %s",
+					imageVersion, arch,
+				)
+			}
+
+			logrus.Infof("Digest for %s on %s: %s", imageVersion, arch, digest)
+		}
+	}
+
+	return true, nil
+}
+
 func (i *Images) getManifestImages(
 	registry, version, buildPath string,
 	forTarballFn func(path, origTag, newTagWithArch string) error,