sync-tags: add support for v0.x.y tags

Implements changes to the KEP in kubernetes/enhancements#1350.

This commit updates sync-tags to publish `v0.x.y` tags for corresponding
`v1.x.y` Kubernetes tag. The old `kubernetes-1.x.y` tags are still also
published. Both `kubernetes-1.x.y` and `v0.x.y` point to the same sha.

For `v0.x.y`:

- At this tag, `go.mod` uses `v0.x.y` tags for dependencies.
- When this tag is manually specified in a `go.mod` file, it will be
retained as `v0.x.y`.

For `kubernetes-1.x.y`:

- At this tag, `go.mod` uses `v0.x.y` tags for dependencies. Note: this
differs from the previous way, where pseudoversions were used.
- When this tag is manually specified in a `go.mod` file, go will resolve
it to the appropriate pseudoversion.

artifacts/scripts/construct.sh

@@ -150,6 +150,7 @@ if [ -z "${SKIP_TAGS}" ]; then
                --dependencies "${DEPS}" \
                --mapping-output-file "../tag-${REPO}-{{.Tag}}-mapping" \
                --generate-godeps=${PUBLISHER_BOT_GENERATE_GODEPS:-false} \
+               --publish-v0-semver \
                -alsologtostderr \
                "${EXTRA_ARGS[@]-}"
     if [ "${LAST_HEAD}" != "$(git rev-parse ${LAST_BRANCH})" ]; then

cmd/sync-tags/gomod.go

@@ -33,7 +33,7 @@ import (
 
 // updateGomodWithTaggedDependencies gets the dependencies at the given tag and fills go.mod and go.sum.
 // If anything is changed, it commits the changes. Returns true if go.mod changed.
-func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, error) {
+func updateGomodWithTaggedDependencies(tag string, depsRepo []string, semverTag bool) (bool, error) {
 	found := map[string]bool{}
 	changed := false
 
@@ -59,10 +59,14 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err
 			return changed, fmt.Errorf("failed to get tag %s for %q: %v", tag, depPkg, err)
 		}
 		rev := commit.String()
-		pseudoVersion := fmt.Sprintf("v0.0.0-%s-%s", commitTime.UTC().Format("20060102150405"), rev[:12])
+		pseudoVersionOrTag := fmt.Sprintf("v0.0.0-%s-%s", commitTime.UTC().Format("20060102150405"), rev[:12])
 
-		// in case the pseudoVersion has not changed, running go mod download will help
-		// in avoiding packaging it up if the pseudoVersion has been published already
+		if semverTag {
+			pseudoVersionOrTag = tag
+		}
+
+		// in case the pseudoVersion/tag has not changed, running go mod download will help
+		// in avoiding packaging it up if the pseudoVersion/tag has been published already
 		downloadCommand := exec.Command("go", "mod", "download")
 		downloadCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
 		downloadCommand.Stdout = os.Stdout
@@ -71,46 +75,46 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err
 			return changed, fmt.Errorf("error running go mod download for %s: %v", depPkg, err)
 		}
 
-		// check if we have the pseudoVersion published already. if we don't, package it up
+		// check if we have the pseudoVersion/tag published already. if we don't, package it up
 		// and save to local mod download cache.
-		if err := packageDepToGoModCache(depPath, depPkg, rev, pseudoVersion, commitTime); err != nil {
+		if err := packageDepToGoModCache(depPath, depPkg, rev, pseudoVersionOrTag, commitTime); err != nil {
 			return changed, fmt.Errorf("failed to package %s dependency: %v", depPkg, err)
 		}
 
-		requireCommand := exec.Command("go", "mod", "edit", "-fmt", "-require", fmt.Sprintf("%s@%s", depPkg, pseudoVersion))
+		requireCommand := exec.Command("go", "mod", "edit", "-fmt", "-require", fmt.Sprintf("%s@%s", depPkg, pseudoVersionOrTag))
 		requireCommand.Env = append(os.Environ(), "GO111MODULE=on")
 		requireCommand.Stdout = os.Stdout
 		requireCommand.Stderr = os.Stderr
 		if err := requireCommand.Run(); err != nil {
-			return changed, fmt.Errorf("unable to pin %s in the require section of go.mod to %s: %v", depPkg, pseudoVersion, err)
+			return changed, fmt.Errorf("unable to pin %s in the require section of go.mod to %s: %v", depPkg, pseudoVersionOrTag, err)
 		}
 
-		replaceCommand := exec.Command("go", "mod", "edit", "-fmt", "-replace", fmt.Sprintf("%s=%s@%s", depPkg, depPkg, pseudoVersion))
+		replaceCommand := exec.Command("go", "mod", "edit", "-fmt", "-replace", fmt.Sprintf("%s=%s@%s", depPkg, depPkg, pseudoVersionOrTag))
 		replaceCommand.Env = append(os.Environ(), "GO111MODULE=on")
 		replaceCommand.Stdout = os.Stdout
 		replaceCommand.Stderr = os.Stderr
 		if err := replaceCommand.Run(); err != nil {
-			return changed, fmt.Errorf("unable to pin %s in the replace section of go.mod to %s: %v", depPkg, pseudoVersion, err)
+			return changed, fmt.Errorf("unable to pin %s in the replace section of go.mod to %s: %v", depPkg, pseudoVersionOrTag, err)
 		}
 
 		downloadCommand2 := exec.Command("go", "mod", "download")
 		downloadCommand2.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
 		downloadCommand2.Stdout = os.Stdout
 		downloadCommand2.Stderr = os.Stderr
 		if err := downloadCommand2.Run(); err != nil {
-			return changed, fmt.Errorf("error running go mod download for pseudo-version %s for %s: %v", pseudoVersion, depPkg, err)
+			return changed, fmt.Errorf("error running go mod download for pseudo-version %s for %s: %v", pseudoVersionOrTag, depPkg, err)
 		}
 
 		tidyCommand := exec.Command("go", "mod", "tidy")
-		tidyCommand.Env = append(os.Environ(), "GO111MODULE=on", "GOPOXY=file://${GOPATH}/pkg/mod/cache/download")
+		tidyCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPROXY=file://%s/pkg/mod/cache/download", os.Getenv("GOPATH")))
 		tidyCommand.Stdout = os.Stdout
 		tidyCommand.Stderr = os.Stderr
 		if err := tidyCommand.Run(); err != nil {
 			return changed, fmt.Errorf("unable to run go mod tidy for %s at %s: %v", depPkg, rev, err)
 		}
 
 		found[dep] = true
-		fmt.Printf("Bumping %s in go.mod to %s\n.", depPkg, rev)
+		fmt.Printf("Bumping %s in go.mod to %s.\n", depPkg, rev)
 		changed = true
 	}
 
@@ -146,18 +150,18 @@ type ModuleInfo struct {
 	Time    string
 }
 
-func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commitTime time.Time) error {
+func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersionOrTag string, commitTime time.Time) error {
 	cacheDir := fmt.Sprintf("%s/pkg/mod/cache/download/%s/@v", os.Getenv("GOPATH"), depPkg)
-	goModFile := fmt.Sprintf("%s/%s.mod", cacheDir, pseudoVersion)
+	goModFile := fmt.Sprintf("%s/%s.mod", cacheDir, pseudoVersionOrTag)
 
 	if _, err := os.Stat(goModFile); err == nil {
-		fmt.Printf("Pseudo version %s for %s is already packaged up.\n", pseudoVersion, depPkg)
+		fmt.Printf("%s for %s is already packaged up.\n", pseudoVersionOrTag, depPkg)
 		return nil
 	} else if err != nil && !os.IsNotExist(err) {
 		return fmt.Errorf("Could not check if %s exists: %v", goModFile, err)
 	}
 
-	fmt.Printf("Packaging up pseudo version %s for %s into go mod cache.\n", pseudoVersion, depPkg)
+	fmt.Printf("Packaging up %s for %s into go mod cache.\n", pseudoVersionOrTag, depPkg)
 
 	// create the cache if it doesn't exist
 	if err := os.MkdirAll(filepath.Dir(goModFile), os.FileMode(755)); err != nil {
@@ -173,14 +177,14 @@ func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commi
 		return fmt.Errorf("failed to checkout %s at %s: %v", depPkg, commit, err)
 	}
 
-	// copy go.mod to pseudoVersion.mod in the cache dir
+	// copy go.mod to the cache dir
 	if err := copyFile(fmt.Sprintf("%s/go.mod", depPath), goModFile); err != nil {
 		return fmt.Errorf("unable to copy %s file to %s to gomod cache for %s: %v", fmt.Sprintf("%s/go.mod", depPath), goModFile, depPkg, err)
 	}
 
-	// create pseudoVersion.info file in the cache dir
+	// create info file in the cache dir
 	moduleInfo := ModuleInfo{
-		Version: pseudoVersion,
+		Version: pseudoVersionOrTag,
 		Name:    commit,
 		Short:   commit[:12],
 		Time:    commitTime.UTC().Format("2006-01-02T15:04:05Z"),
@@ -190,17 +194,17 @@ func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commi
 	if err != nil {
 		return fmt.Errorf("error marshaling .info file for %s: %v", depPkg, err)
 	}
-	if err := ioutil.WriteFile(fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersion), moduleFile, 0644); err != nil {
-		return fmt.Errorf("failed to write %s file for %s: %v", fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersion), depPkg, err)
+	if err := ioutil.WriteFile(fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), moduleFile, 0644); err != nil {
+		return fmt.Errorf("failed to write %s file for %s: %v", fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), depPkg, err)
 	}
 
-	// create the pseudoVersion.zip file in the cache dir. This zip file has the same hash
+	// create the zip file in the cache dir. This zip file has the same hash
 	// as of the zip file that would have been created by go mod download.
-	zipCommand := exec.Command("/gomod-zip", "--package-name", depPkg, "--pseudo-version", pseudoVersion)
+	zipCommand := exec.Command("/gomod-zip", "--package-name", depPkg, "--pseudo-version", pseudoVersionOrTag)
 	zipCommand.Stdout = os.Stdout
 	zipCommand.Stderr = os.Stderr
 	if err := zipCommand.Run(); err != nil {
-		return fmt.Errorf("failed to run gomod-zip for %s at %s: %v", depPkg, pseudoVersion, err)
+		return fmt.Errorf("failed to run gomod-zip for %s at %s: %v", depPkg, pseudoVersionOrTag, err)
 	}
 
 	// append the pseudoVersion to the list file in the cache dir
@@ -210,7 +214,7 @@ func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commi
 	}
 	defer listFile.Close()
 
-	if _, err := listFile.WriteString(fmt.Sprintf("%s\n", pseudoVersion)); err != nil {
+	if _, err := listFile.WriteString(fmt.Sprintf("%s\n", pseudoVersionOrTag)); err != nil {
 		return fmt.Errorf("unable to write to list file in %s: %v", cacheDir, err)
 	}
 

cmd/sync-tags/main.go

@@ -28,6 +28,7 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/blang/semver"
 	"github.com/golang/glog"
 	"github.com/renstrom/dedent"
 	gogit "gopkg.in/src-d/go-git.v4"
@@ -73,6 +74,7 @@ func main() {
 	skipFetch := flag.Bool("skip-fetch", false, "skip fetching tags")
 	generateGodeps := flag.Bool("generate-godeps", false, "regenerate Godeps.json from go.mod")
 	mappingOutputFile := flag.String("mapping-output-file", "", "a file name to write the source->dest hash mapping to ({{.Tag}} is substituted with the tag name, {{.Branch}} with the local branch name)")
+	publishSemverTags := flag.Bool("publish-v0-semver", false, "publish v0.x.y tag at destination repo for v1.x.y tag at the source repo")
 
 	flag.Usage = Usage
 	flag.Parse()
@@ -187,6 +189,19 @@ func main() {
 			bName = *prefix + name[1:] // remove the v
 		}
 
+		// check if name is a semver tag
+		// the semver lib requires the v prefix to be dropped before parsing
+		var skipPublishingSemverTag bool
+		if _, err := semver.Parse(name[1:]); err != nil {
+			skipPublishingSemverTag = true
+		}
+
+		// replace v1.x.y with v0.x.y
+		semverTag := name
+		if strings.HasPrefix(name, "v1.") {
+			semverTag = "v0." + strings.TrimPrefix(name, "v1.")
+		}
+
 		// ignore non-annotated tags
 		tag, err := r.TagObject(kh)
 		if err != nil {
@@ -199,13 +214,19 @@ func main() {
 			continue
 		}
 
-		// skip if it already exists in origin
-		if _, found := bTagCommits[bName]; found {
-			continue
-		}
+		// check if the tags exist either at origin or locally
+		_, nonSemverTagAtOrigin := bTagCommits[bName]
+		nonSemverTagExists := nonSemverTagAtOrigin || tagExists(r, bName)
 
-		// do not override tags (we build master first, i.e. the x.y.z-alpha.0 tag on master will not be created for feature branches)
-		if tagExists(r, bName) {
+		_, semverTagAtOrigin := bTagCommits[semverTag]
+		semverTagExists := semverTagAtOrigin || tagExists(r, semverTag)
+
+		// skip tags if they exist locally or at origin already
+		// we want to ensure to not override tags
+		// (we build master first, i.e. the x.y.z-alpha.0 tag on master will not be created for feature branches)
+		if *publishSemverTags && !skipPublishingSemverTag && semverTagExists && nonSemverTagExists {
+			continue
+		} else if nonSemverTagExists {
 			continue
 		}
 
@@ -258,71 +279,69 @@ func main() {
 		// update go.mod or Godeps.json to point to actual tagged version in the dependencies. This version might differ
 		// from the one currently in go.mod  or Godeps.json because the other repo could have gotten more commit for this
 		// tag, but this repo didn't. Compare https://github.com/kubernetes/publishing-bot/issues/12 for details.
+		var changed, goModExists bool
+		_, err = os.Stat("go.mod")
+		if err == nil {
+			goModExists = true
+		}
+
 		if len(dependentRepos) > 0 {
-			fmt.Printf("Checking that dependencies point to the actual tags in %s.\n", strings.Join(dependentRepos, ", "))
-			wt, err := r.Worktree()
-			if err != nil {
-				glog.Fatalf("Failed to get working tree: %v", err)
-			}
-			fmt.Printf("Checking out branch tag commit %s.\n", bh.String())
-			if err := wt.Checkout(&gogit.CheckoutOptions{Hash: bh}); err != nil {
-				glog.Fatalf("Failed to checkout %v: %v", bh, err)
-			}
+			wt := checkoutBranchTagCommit(r, bh, dependentRepos)
 
 			// if go.mod exists, fix only go.mod and generate Godeps.json from it later
 			// if it doesn't exist, check if Godeps.json exists, and update it
-			var changed, goModChanged bool
-			_, err = os.Stat("go.mod")
-			if os.IsNotExist(err) {
+			if !goModExists {
 				if _, err2 := os.Stat("Godeps/Godeps.json"); err2 == nil {
 					fmt.Printf("Updating Godeps.json to point to %s tag.\n", bName)
 					changed, err = updateGodepsJsonWithTaggedDependencies(bName, dependentRepos)
 					if err != nil {
 						glog.Fatalf("Failed to update Godeps.json for tag %s: %v", bName, err)
 					}
 				}
-			} else if err == nil {
-				fmt.Printf("Updating go.mod and go.sum to point to %s tag.\n", bName)
-				changed, err = updateGomodWithTaggedDependencies(bName, dependentRepos)
-				if err != nil {
-					glog.Fatalf("Failed to update go.mod and go.sum for tag %s: %v", bName, err)
-				}
-				goModChanged = true
-			}
-
-			if goModChanged && *generateGodeps {
-				fmt.Printf("Regenerating Godeps.json from go.mod.\n")
-				if err := regenerateGodepsFromGoMod(); err != nil {
-					glog.Fatalf("Failed to regenerate Godeps.json from go.mod: %v", err)
+			} else {
+				if *publishSemverTags && !skipPublishingSemverTag {
+					changed = updateGoModAndGodeps(semverTag, dependentRepos, *generateGodeps, true)
+				} else {
+					changed = updateGoModAndGodeps(bName, dependentRepos, *generateGodeps, false)
 				}
 			}
 
 			if changed {
-				fmt.Printf("Adding extra commit fixing dependencies to point to %s tags.\n", bName)
-				publishingBotNow := publishingBot
-				publishingBotNow.When = time.Now()
-				bh, err = wt.Commit(fmt.Sprintf("Fix dependencies to point to %s tag", bName), &gogit.CommitOptions{
-					All:       true,
-					Author:    &publishingBotNow,
-					Committer: &publishingBotNow,
-				})
-				if err != nil {
-					glog.Fatalf("Failed to commit changes to fix dependencies to point to %s tag: %v", bName, err)
+				if *publishSemverTags && !skipPublishingSemverTag {
+					bh = createCommitToFixDeps(wt, semverTag)
+				} else {
+					bh = createCommitToFixDeps(wt, bName)
 				}
 			}
 		}
 
-		// create prefixed annotated tag
-		fmt.Printf("Tagging %v as %q.\n", bh, bName)
-		err = createAnnotatedTag(bh, bName, tag.Tagger.When, dedent.Dedent(fmt.Sprintf(`
+		// create semver annotated tag
+		if *publishSemverTags && !skipPublishingSemverTag && !semverTagExists {
+			fmt.Printf("Tagging %v as %q.\n", bh, semverTag)
+			err = createAnnotatedTag(bh, semverTag, tag.Tagger.When, dedent.Dedent(fmt.Sprintf(`
 			Kubernetes release %s
 
 			Based on https://github.com/kubernetes/kubernetes/releases/tag/%s
 			`, name, name)))
-		if err != nil {
-			glog.Fatalf("Failed to create tag %q: %v", bName, err)
+			if err != nil {
+				glog.Fatalf("Failed to create tag %q: %v", semverTag, err)
+			}
+			createdTags = append(createdTags, semverTag)
+		}
+
+		// create non-semver prefixed annotated tag
+		if !nonSemverTagExists {
+			fmt.Printf("Tagging %v as %q.\n", bh, bName)
+			err = createAnnotatedTag(bh, bName, tag.Tagger.When, dedent.Dedent(fmt.Sprintf(`
+				Kubernetes release %s
+
+				Based on https://github.com/kubernetes/kubernetes/releases/tag/%s
+				`, name, name)))
+			if err != nil {
+				glog.Fatalf("Failed to create tag %q: %v", bName, err)
+			}
+			createdTags = append(createdTags, bName)
 		}
-		createdTags = append(createdTags, bName)
 	}
 
 	// write push command for new tags
@@ -463,7 +482,7 @@ func mappingOutputFileName(fnameTpl string, branch, tag string) string {
 
 func regenerateGodepsFromGoMod() error {
 	goListCommand := exec.Command("go", "list", "-m", "-json", "all")
-	goListCommand.Env = append(os.Environ(), "GO111MODULE=on", "GOPOXY=file://${GOPATH}/pkg/mod/cache/download")
+	goListCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPROXY=file://%s/pkg/mod/cache/download", os.Getenv("GOPATH")))
 	goMod, err := goListCommand.Output()
 	if err != nil {
 		return fmt.Errorf("Failed to get output of go list -m -json all: %v", err)
@@ -488,3 +507,48 @@ func regenerateGodepsFromGoMod() error {
 	}
 	return nil
 }
+
+func checkoutBranchTagCommit(r *gogit.Repository, bh plumbing.Hash, dependentRepos []string) *gogit.Worktree {
+	fmt.Printf("Checking that dependencies point to the actual tags in %s.\n", strings.Join(dependentRepos, ", "))
+	wt, err := r.Worktree()
+	if err != nil {
+		glog.Fatalf("Failed to get working tree: %v", err)
+	}
+
+	fmt.Printf("Checking out branch tag commit %s.\n", bh.String())
+	if err := wt.Checkout(&gogit.CheckoutOptions{Hash: bh}); err != nil {
+		glog.Fatalf("Failed to checkout %v: %v", bh, err)
+	}
+	return wt
+}
+
+func updateGoModAndGodeps(tag string, dependentRepos []string, generateGodeps, publishSemverTags bool) bool {
+	fmt.Printf("Updating go.mod and go.sum to point to %s tag.\n", tag)
+	changed, err := updateGomodWithTaggedDependencies(tag, dependentRepos, publishSemverTags)
+	if err != nil {
+		glog.Fatalf("Failed to update go.mod and go.sum for tag %s: %v", tag, err)
+	}
+
+	if changed && generateGodeps {
+		fmt.Printf("Regenerating Godeps.json from go.mod.\n")
+		if err := regenerateGodepsFromGoMod(); err != nil {
+			glog.Fatalf("Failed to regenerate Godeps.json from go.mod: %v", err)
+		}
+	}
+	return changed
+}
+
+func createCommitToFixDeps(wt *gogit.Worktree, tag string) plumbing.Hash {
+	fmt.Printf("Adding extra commit to update dependencies to %s tag.\n", tag)
+	publishingBotNow := publishingBot
+	publishingBotNow.When = time.Now()
+	bh, err := wt.Commit(fmt.Sprintf("Update dependencies to %s tag", tag), &gogit.CommitOptions{
+		All:       true,
+		Author:    &publishingBotNow,
+		Committer: &publishingBotNow,
+	})
+	if err != nil {
+		glog.Fatalf("Failed to commit changes to update dependencies to %s tag: %v", tag, err)
+	}
+	return bh
+}