Allow staging via krel for kubetest

This adds a new `--use-krel` flag to kubetest to indicate that we want to stage the sources directly via the API provided by k/release rather than the deprecated push-build.sh script (lives in k/release, too). The idea is to add another job later on for testing this flag on a non critical Kubernetes build+push job (should live in sig-release). Signed-off-by: Sascha Grunert <[email protected]> Signed-off-by: Sascha Grunert <[email protected]>
kubernetes · Jan 20, 2021 · 052daf3 · 052daf3
1 parent 3579070
commit 052daf3
Show file tree

Hide file tree

Showing 8 changed files with 670 additions and 100 deletions.
diff --git a/go.mod b/go.mod
@@ -59,7 +59,7 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.6.6
 	github.com/influxdata/influxdb v0.0.0-20161215172503-049f9b42e9a5
 	github.com/jinzhu/gorm v1.9.12
-	github.com/klauspost/pgzip v1.2.1
+	github.com/klauspost/pgzip v1.2.5
 	github.com/mattn/go-zglob v0.0.2
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
 	github.com/pelletier/go-toml v1.8.0
@@ -70,7 +70,7 @@ require (
 	github.com/prometheus/common v0.10.0
 	github.com/satori/go.uuid v1.2.0
 	github.com/shurcooL/githubv4 v0.0.0-20191102174205-af46314aec7b
-	github.com/sirupsen/logrus v1.6.0
+	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
 	github.com/tektoncd/pipeline v0.13.1-0.20200625065359-44f22a067b75
@@ -79,11 +79,11 @@ require (
 	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b
 	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
-	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
+	golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
-	golang.org/x/tools v0.0.0-20200918232735-d647fc253266
-	google.golang.org/api v0.32.0
+	golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd
+	google.golang.org/api v0.35.0
 	gopkg.in/fsnotify.v1 v1.4.7
 	gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5
 	gopkg.in/yaml.v2 v2.3.0
@@ -94,6 +94,7 @@ require (
 	k8s.io/code-generator v0.20.2
 	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.4.0
+	k8s.io/release v0.6.0
 	k8s.io/utils v0.0.0-20210111153108-fddb29f9d009
 	knative.dev/pkg v0.0.0-20200711004937-22502028e31a
 	mvdan.cc/xurls/v2 v2.0.0

diff --git a/go.sum b/go.sum
diff --git a/kubetest/BUILD.bazel b/kubetest/BUILD.bazel
@@ -43,9 +43,12 @@ go_library(
         "@com_github_docker_docker//api/types:go_default_library",
         "@com_github_docker_docker//client:go_default_library",
         "@com_github_pelletier_go_toml//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_satori_go_uuid//:go_default_library",
         "@com_github_spf13_pflag//:go_default_library",
         "@com_google_cloud_go_storage//:go_default_library",
+        "@io_k8s_release//pkg/build:go_default_library",
+        "@io_k8s_release//pkg/object:go_default_library",
         "@io_k8s_sigs_boskos//client:go_default_library",
         "@org_golang_x_crypto//ssh:go_default_library",
     ],

diff --git a/kubetest/main.go b/kubetest/main.go
@@ -99,7 +99,6 @@ type options struct {
 	kubemarkNodes           string // TODO(fejta): switch to int after migration
 	logexporterGCSPath      string
 	metadataSources         string
-	noAllowDup              bool
 	nodeArgs                string
 	nodeTestArgs            string
 	nodeTests               bool
@@ -168,7 +167,7 @@ func defineFlags() *options {
 	flag.StringVar(&o.metadataSources, "metadata-sources", "images.json", "Comma-separated list of files inside ./artifacts to merge into metadata.json")
 	flag.StringVar(&o.nodeArgs, "node-args", "", "Args for node e2e tests.")
 	flag.StringVar(&o.nodeTestArgs, "node-test-args", "", "Test args specifically for node e2e tests.")
-	flag.BoolVar(&o.noAllowDup, "no-allow-dup", false, "if set --allow-dup will not be passed to push-build and --stage will error if the build already exists on the gcs path")
+	flag.BoolVar(&o.stage.noAllowDup, "no-allow-dup", false, "if set --allow-dup will not be passed to push-build and --stage will error if the build already exists on the gcs path")
 	flag.BoolVar(&o.nodeTests, "node-tests", false, "If true, run node-e2e tests.")
 	flag.StringVar(&o.provider, "provider", "", "Kubernetes provider such as gce, gke, aws, etc")
 	flag.StringVar(&o.publish, "publish", "", "Publish version to the specified gs:// path on success")
@@ -189,6 +188,7 @@ func defineFlags() *options {
 	flag.BoolVar(&o.up, "up", false, "If true, start the e2e cluster. If cluster is already up, recreate it.")
 	flag.StringVar(&o.upgradeArgs, "upgrade_args", "", "If set, run upgrade tests before other tests")
 	flag.DurationVar(&o.boskosWaitDuration, "boskos-wait-duration", 5*time.Minute, "Defines how long it waits until quit getting Boskos resoure, default 5 minutes")
+	flag.BoolVar(&o.stage.useKrel, "use-krel", false, "If true, use the Kubernetes Release Toolbox (krel). Works only for staging right now.")
 
 	// The "-v" flag was also used by glog, which is used by k8s.io/client-go. Duplicate flags cause panics.
 	// 1. Even if we could convince glog to change, they have too many consumers to ever do so.
@@ -435,7 +435,7 @@ func acquireKubernetes(o *options, d deployer) error {
 	// Potentially stage build binaries somewhere on GCS
 	if o.stage.Enabled() {
 		if err := control.XMLWrap(&suite, "Stage", func() error {
-			return o.stage.Stage(o.noAllowDup)
+			return o.stage.Stage()
 		}); err != nil {
 			return err
 		}

diff --git a/kubetest/stage.go b/kubetest/stage.go
@@ -22,6 +22,10 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/pkg/errors"
+	"k8s.io/release/pkg/build"
+	"k8s.io/release/pkg/object"
+
 	"k8s.io/test-infra/kubetest/util"
 )
 
@@ -31,15 +35,20 @@ type stageStrategy struct {
 	gcsSuffix      string
 	versionSuffix  string
 	dockerRegistry string
+	noAllowDup     bool
+	useKrel        bool
 }
 
 // Return something like gs://bucket/ci/suffix
 func (s *stageStrategy) String() string {
-	p := "devel"
+	return fmt.Sprintf("%v%v%v", s.bucket, s.releaseType(), s.gcsSuffix)
+}
+
+func (s *stageStrategy) releaseType() string {
 	if s.ci {
-		p = "ci"
+		return "ci"
 	}
-	return fmt.Sprintf("%v%v%v", s.bucket, p, s.gcsSuffix)
+	return "devel"
 }
 
 // Parse bucket, ci, suffix from gs://BUCKET/ci/SUFFIX
@@ -65,15 +74,36 @@ func (s *stageStrategy) Enabled() bool {
 }
 
 // Stage the release build to GCS.
-// Essentially release/push-build.sh --bucket=B --ci? --gcs-suffix=S --noupdatelatest
-func (s *stageStrategy) Stage(noAllowDup bool) error {
+func (s *stageStrategy) Stage() error {
+	// Trim the gcs prefix from the bucket
+	s.bucket = strings.TrimPrefix(s.bucket, object.GcsPrefix)
+
+	// Essentially release/push-build.sh --bucket=B --ci? --gcs-suffix=S
+	// --noupdatelatest
+	if !s.useKrel {
+		return errors.Wrap(s.stageViaPushBuildSh(), "stage via push-build.sh")
+	}
+
+	return errors.Wrap(
+		build.NewInstance(&build.Options{
+			Bucket:         s.bucket,
+			Registry:       s.dockerRegistry,
+			GCSRoot:        s.gcsSuffix,
+			VersionSuffix:  s.versionSuffix,
+			AllowDup:       !s.noAllowDup,
+			CI:             s.ci,
+			NoUpdateLatest: true,
+		}).Push(), "stage via krel push",
+	)
+}
+
+func (s *stageStrategy) stageViaPushBuildSh() error {
 	name := util.K8s("release", "push-build.sh")
-	b := strings.TrimPrefix(s.bucket, "gs://")
 	args := []string{
 		"--nomock",
 		"--verbose",
 		"--noupdatelatest", // we may need to expose control of this if build jobs start using kubetest
-		fmt.Sprintf("--bucket=%v", b),
+		fmt.Sprintf("--bucket=%v", s.bucket),
 	}
 	if s.ci {
 		args = append(args, "--ci")
@@ -88,7 +118,7 @@ func (s *stageStrategy) Stage(noAllowDup bool) error {
 		args = append(args, fmt.Sprintf("--docker-registry=%s", s.dockerRegistry))
 	}
 
-	if !noAllowDup {
+	if !s.noAllowDup {
 		args = append(args, "--allow-dup")
 	}
 

diff --git a/prow/test/BUILD.bazel b/prow/test/BUILD.bazel
@@ -23,6 +23,7 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//prow/test/data:all-srcs",
+        "//prow/test/integration/test:all-srcs",
     ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],

diff --git a/prow/test/integration/test/BUILD.bazel b/prow/test/integration/test/BUILD.bazel
@@ -43,11 +43,8 @@ go_library(
     srcs = ["setup.go"],
     importpath = "k8s.io/test-infra/prow/test/integration/test",
     deps = [
-        "@io_k8s_client_go//kubernetes:go_default_library",
         "@io_k8s_client_go//plugin/pkg/client/auth/oidc:go_default_library",
-        "@io_k8s_client_go//rest:go_default_library",
         "@io_k8s_client_go//tools/clientcmd:go_default_library",
         "@io_k8s_sigs_controller_runtime//pkg/client:go_default_library",
-        "@io_k8s_sigs_controller_runtime//pkg/manager:go_default_library",
     ],
 )