diff --git a/artifacts/scripts/construct.sh b/artifacts/scripts/construct.sh index 178460cbb..c7d0ceeb6 100755 --- a/artifacts/scripts/construct.sh +++ b/artifacts/scripts/construct.sh @@ -150,6 +150,7 @@ if [ -z "${SKIP_TAGS}" ]; then --dependencies "${DEPS}" \ --mapping-output-file "../tag-${REPO}-{{.Tag}}-mapping" \ --generate-godeps=${PUBLISHER_BOT_GENERATE_GODEPS:-false} \ + --publish-semver-tags \ -alsologtostderr \ "${EXTRA_ARGS[@]-}" if [ "${LAST_HEAD}" != "$(git rev-parse ${LAST_BRANCH})" ]; then diff --git a/cmd/sync-tags/gomod.go b/cmd/sync-tags/gomod.go index 28d783f52..a065582c4 100644 --- a/cmd/sync-tags/gomod.go +++ b/cmd/sync-tags/gomod.go @@ -33,7 +33,7 @@ import ( // updateGomodWithTaggedDependencies gets the dependencies at the given tag and fills go.mod and go.sum. // If anything is changed, it commits the changes. Returns true if go.mod changed. -func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, error) { +func updateGomodWithTaggedDependencies(tag string, depsRepo []string, semverTag bool) (bool, error) { found := map[string]bool{} changed := false @@ -59,10 +59,14 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err return changed, fmt.Errorf("failed to get tag %s for %q: %v", tag, depPkg, err) } rev := commit.String() - pseudoVersion := fmt.Sprintf("v0.0.0-%s-%s", commitTime.UTC().Format("20060102150405"), rev[:12]) + pseudoVersionOrTag := fmt.Sprintf("v0.0.0-%s-%s", commitTime.UTC().Format("20060102150405"), rev[:12]) - // in case the pseudoVersion has not changed, running go mod download will help - // in avoiding packaging it up if the pseudoVersion has been published already + if semverTag { + pseudoVersionOrTag = tag + } + + // in case the pseudoVersion/tag has not changed, running go mod download will help + // in avoiding packaging it up if the pseudoVersion/tag has been published already downloadCommand := exec.Command("go", "mod", "download") downloadCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org") downloadCommand.Stdout = os.Stdout @@ -71,26 +75,26 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err return changed, fmt.Errorf("error running go mod download for %s: %v", depPkg, err) } - // check if we have the pseudoVersion published already. if we don't, package it up + // check if we have the pseudoVersion/tag published already. if we don't, package it up // and save to local mod download cache. - if err := packageDepToGoModCache(depPath, depPkg, rev, pseudoVersion, commitTime); err != nil { + if err := packageDepToGoModCache(depPath, depPkg, rev, pseudoVersionOrTag, commitTime); err != nil { return changed, fmt.Errorf("failed to package %s dependency: %v", depPkg, err) } - requireCommand := exec.Command("go", "mod", "edit", "-fmt", "-require", fmt.Sprintf("%s@%s", depPkg, pseudoVersion)) + requireCommand := exec.Command("go", "mod", "edit", "-fmt", "-require", fmt.Sprintf("%s@%s", depPkg, pseudoVersionOrTag)) requireCommand.Env = append(os.Environ(), "GO111MODULE=on") requireCommand.Stdout = os.Stdout requireCommand.Stderr = os.Stderr if err := requireCommand.Run(); err != nil { - return changed, fmt.Errorf("unable to pin %s in the require section of go.mod to %s: %v", depPkg, pseudoVersion, err) + return changed, fmt.Errorf("unable to pin %s in the require section of go.mod to %s: %v", depPkg, pseudoVersionOrTag, err) } - replaceCommand := exec.Command("go", "mod", "edit", "-fmt", "-replace", fmt.Sprintf("%s=%s@%s", depPkg, depPkg, pseudoVersion)) + replaceCommand := exec.Command("go", "mod", "edit", "-fmt", "-replace", fmt.Sprintf("%s=%s@%s", depPkg, depPkg, pseudoVersionOrTag)) replaceCommand.Env = append(os.Environ(), "GO111MODULE=on") replaceCommand.Stdout = os.Stdout replaceCommand.Stderr = os.Stderr if err := replaceCommand.Run(); err != nil { - return changed, fmt.Errorf("unable to pin %s in the replace section of go.mod to %s: %v", depPkg, pseudoVersion, err) + return changed, fmt.Errorf("unable to pin %s in the replace section of go.mod to %s: %v", depPkg, pseudoVersionOrTag, err) } downloadCommand2 := exec.Command("go", "mod", "download") @@ -98,11 +102,11 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err downloadCommand2.Stdout = os.Stdout downloadCommand2.Stderr = os.Stderr if err := downloadCommand2.Run(); err != nil { - return changed, fmt.Errorf("error running go mod download for pseudo-version %s for %s: %v", pseudoVersion, depPkg, err) + return changed, fmt.Errorf("error running go mod download for pseudo-version %s for %s: %v", pseudoVersionOrTag, depPkg, err) } tidyCommand := exec.Command("go", "mod", "tidy") - tidyCommand.Env = append(os.Environ(), "GO111MODULE=on", "GOPOXY=file://${GOPATH}/pkg/mod/cache/download") + tidyCommand.Env = append(os.Environ(), "GO111MODULE=on") tidyCommand.Stdout = os.Stdout tidyCommand.Stderr = os.Stderr if err := tidyCommand.Run(); err != nil { @@ -110,7 +114,7 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err } found[dep] = true - fmt.Printf("Bumping %s in go.mod to %s\n.", depPkg, rev) + fmt.Printf("Bumping %s in go.mod to %s.\n", depPkg, rev) changed = true } @@ -146,18 +150,18 @@ type ModuleInfo struct { Time string } -func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commitTime time.Time) error { +func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersionOrTag string, commitTime time.Time) error { cacheDir := fmt.Sprintf("%s/pkg/mod/cache/download/%s/@v", os.Getenv("GOPATH"), depPkg) - goModFile := fmt.Sprintf("%s/%s.mod", cacheDir, pseudoVersion) + goModFile := fmt.Sprintf("%s/%s.mod", cacheDir, pseudoVersionOrTag) if _, err := os.Stat(goModFile); err == nil { - fmt.Printf("Pseudo version %s for %s is already packaged up.\n", pseudoVersion, depPkg) + fmt.Printf("%s for %s is already packaged up.\n", pseudoVersionOrTag, depPkg) return nil } else if err != nil && !os.IsNotExist(err) { return fmt.Errorf("Could not check if %s exists: %v", goModFile, err) } - fmt.Printf("Packaging up pseudo version %s for %s into go mod cache.\n", pseudoVersion, depPkg) + fmt.Printf("Packaging up %s for %s into go mod cache.\n", pseudoVersionOrTag, depPkg) // create the cache if it doesn't exist if err := os.MkdirAll(filepath.Dir(goModFile), os.FileMode(755)); err != nil { @@ -173,14 +177,14 @@ func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commi return fmt.Errorf("failed to checkout %s at %s: %v", depPkg, commit, err) } - // copy go.mod to pseudoVersion.mod in the cache dir + // copy go.mod to the cache dir if err := copyFile(fmt.Sprintf("%s/go.mod", depPath), goModFile); err != nil { return fmt.Errorf("unable to copy %s file to %s to gomod cache for %s: %v", fmt.Sprintf("%s/go.mod", depPath), goModFile, depPkg, err) } - // create pseudoVersion.info file in the cache dir + // create info file in the cache dir moduleInfo := ModuleInfo{ - Version: pseudoVersion, + Version: pseudoVersionOrTag, Name: commit, Short: commit[:12], Time: commitTime.UTC().Format("2006-01-02T15:04:05Z"), @@ -190,17 +194,17 @@ func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commi if err != nil { return fmt.Errorf("error marshaling .info file for %s: %v", depPkg, err) } - if err := ioutil.WriteFile(fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersion), moduleFile, 0644); err != nil { - return fmt.Errorf("failed to write %s file for %s: %v", fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersion), depPkg, err) + if err := ioutil.WriteFile(fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), moduleFile, 0644); err != nil { + return fmt.Errorf("failed to write %s file for %s: %v", fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), depPkg, err) } - // create the pseudoVersion.zip file in the cache dir. This zip file has the same hash + // create the zip file in the cache dir. This zip file has the same hash // as of the zip file that would have been created by go mod download. - zipCommand := exec.Command("/gomod-zip", "--package-name", depPkg, "--pseudo-version", pseudoVersion) + zipCommand := exec.Command("/gomod-zip", "--package-name", depPkg, "--pseudo-version", pseudoVersionOrTag) zipCommand.Stdout = os.Stdout zipCommand.Stderr = os.Stderr if err := zipCommand.Run(); err != nil { - return fmt.Errorf("failed to run gomod-zip for %s at %s: %v", depPkg, pseudoVersion, err) + return fmt.Errorf("failed to run gomod-zip for %s at %s: %v", depPkg, pseudoVersionOrTag, err) } // append the pseudoVersion to the list file in the cache dir @@ -210,7 +214,7 @@ func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commi } defer listFile.Close() - if _, err := listFile.WriteString(fmt.Sprintf("%s\n", pseudoVersion)); err != nil { + if _, err := listFile.WriteString(fmt.Sprintf("%s\n", pseudoVersionOrTag)); err != nil { return fmt.Errorf("unable to write to list file in %s: %v", cacheDir, err) } diff --git a/cmd/sync-tags/main.go b/cmd/sync-tags/main.go index 293566c5e..974847994 100644 --- a/cmd/sync-tags/main.go +++ b/cmd/sync-tags/main.go @@ -73,6 +73,7 @@ func main() { skipFetch := flag.Bool("skip-fetch", false, "skip fetching tags") generateGodeps := flag.Bool("generate-godeps", false, "regenerate Godeps.json from go.mod") mappingOutputFile := flag.String("mapping-output-file", "", "a file name to write the source->dest hash mapping to ({{.Tag}} is substituted with the tag name, {{.Branch}} with the local branch name)") + publishSemverTags := flag.Bool("publish-semver-tags", false, "publish v0.x.y tag at destination repo for v1.x.y tag at the source repo") flag.Usage = Usage flag.Parse() @@ -186,6 +187,7 @@ func main() { if *prefix != "" { bName = *prefix + name[1:] // remove the v } + semverTag := name[:1] + "0" + name[2:] // replace v1.x.y with v0.x.y // ignore non-annotated tags tag, err := r.TagObject(kh) @@ -199,13 +201,19 @@ func main() { continue } - // skip if it already exists in origin - if _, found := bTagCommits[bName]; found { - continue - } + // check if the tags exist either at origin or locally + _, nonSemverTagAtOrigin := bTagCommits[bName] + nonSemverTagExists := nonSemverTagAtOrigin || tagExists(r, bName) + + _, semverTagAtOrigin := bTagCommits[semverTag] + semverTagExists := semverTagAtOrigin || tagExists(r, semverTag) - // do not override tags (we build master first, i.e. the x.y.z-alpha.0 tag on master will not be created for feature branches) - if tagExists(r, bName) { + // skip tags if they exist locally or at origin already + // we want to ensure to not override tags + // (we build master first, i.e. the x.y.z-alpha.0 tag on master will not be created for feature branches) + if *publishSemverTags && semverTagExists && nonSemverTagExists { + continue + } else if nonSemverTagExists { continue } @@ -258,22 +266,18 @@ func main() { // update go.mod or Godeps.json to point to actual tagged version in the dependencies. This version might differ // from the one currently in go.mod or Godeps.json because the other repo could have gotten more commit for this // tag, but this repo didn't. Compare https://github.com/kubernetes/publishing-bot/issues/12 for details. + var changed, goModExists bool + _, err = os.Stat("go.mod") + if err == nil { + goModExists = true + } + if len(dependentRepos) > 0 { - fmt.Printf("Checking that dependencies point to the actual tags in %s.\n", strings.Join(dependentRepos, ", ")) - wt, err := r.Worktree() - if err != nil { - glog.Fatalf("Failed to get working tree: %v", err) - } - fmt.Printf("Checking out branch tag commit %s.\n", bh.String()) - if err := wt.Checkout(&gogit.CheckoutOptions{Hash: bh}); err != nil { - glog.Fatalf("Failed to checkout %v: %v", bh, err) - } + wt := checkoutBranchTagCommit(r, bh, dependentRepos) // if go.mod exists, fix only go.mod and generate Godeps.json from it later // if it doesn't exist, check if Godeps.json exists, and update it - var changed, goModChanged bool - _, err = os.Stat("go.mod") - if os.IsNotExist(err) { + if !goModExists { if _, err2 := os.Stat("Godeps/Godeps.json"); err2 == nil { fmt.Printf("Updating Godeps.json to point to %s tag.\n", bName) changed, err = updateGodepsJsonWithTaggedDependencies(bName, dependentRepos) @@ -281,48 +285,50 @@ func main() { glog.Fatalf("Failed to update Godeps.json for tag %s: %v", bName, err) } } - } else if err == nil { - fmt.Printf("Updating go.mod and go.sum to point to %s tag.\n", bName) - changed, err = updateGomodWithTaggedDependencies(bName, dependentRepos) - if err != nil { - glog.Fatalf("Failed to update go.mod and go.sum for tag %s: %v", bName, err) - } - goModChanged = true - } - - if goModChanged && *generateGodeps { - fmt.Printf("Regenerating Godeps.json from go.mod.\n") - if err := regenerateGodepsFromGoMod(); err != nil { - glog.Fatalf("Failed to regenerate Godeps.json from go.mod: %v", err) + } else { + if *publishSemverTags { + changed = updateGoModAndGodeps(semverTag, dependentRepos, *generateGodeps, true) + } else { + changed = updateGoModAndGodeps(bName, dependentRepos, *generateGodeps, false) } } if changed { - fmt.Printf("Adding extra commit fixing dependencies to point to %s tags.\n", bName) - publishingBotNow := publishingBot - publishingBotNow.When = time.Now() - bh, err = wt.Commit(fmt.Sprintf("Fix dependencies to point to %s tag", bName), &gogit.CommitOptions{ - All: true, - Author: &publishingBotNow, - Committer: &publishingBotNow, - }) - if err != nil { - glog.Fatalf("Failed to commit changes to fix dependencies to point to %s tag: %v", bName, err) + if *publishSemverTags { + bh = createCommitToFixDeps(wt, semverTag) + } else { + bh = createCommitToFixDeps(wt, bName) } } } - // create prefixed annotated tag - fmt.Printf("Tagging %v as %q.\n", bh, bName) - err = createAnnotatedTag(bh, bName, tag.Tagger.When, dedent.Dedent(fmt.Sprintf(` + // create semver annotated tag + if *publishSemverTags && !semverTagExists { + fmt.Printf("Tagging %v as %q.\n", bh, semverTag) + err = createAnnotatedTag(bh, semverTag, tag.Tagger.When, dedent.Dedent(fmt.Sprintf(` Kubernetes release %s Based on https://github.com/kubernetes/kubernetes/releases/tag/%s `, name, name))) - if err != nil { - glog.Fatalf("Failed to create tag %q: %v", bName, err) + if err != nil { + glog.Fatalf("Failed to create tag %q: %v", semverTag, err) + } + createdTags = append(createdTags, semverTag) + } + + // create non-semver prefixed annotated tag + if !nonSemverTagExists { + fmt.Printf("Tagging %v as %q.\n", bh, bName) + err = createAnnotatedTag(bh, bName, tag.Tagger.When, dedent.Dedent(fmt.Sprintf(` + Kubernetes release %s + + Based on https://github.com/kubernetes/kubernetes/releases/tag/%s + `, name, name))) + if err != nil { + glog.Fatalf("Failed to create tag %q: %v", bName, err) + } + createdTags = append(createdTags, bName) } - createdTags = append(createdTags, bName) } // write push command for new tags @@ -463,7 +469,7 @@ func mappingOutputFileName(fnameTpl string, branch, tag string) string { func regenerateGodepsFromGoMod() error { goListCommand := exec.Command("go", "list", "-m", "-json", "all") - goListCommand.Env = append(os.Environ(), "GO111MODULE=on", "GOPOXY=file://${GOPATH}/pkg/mod/cache/download") + goListCommand.Env = append(os.Environ(), "GO111MODULE=on") goMod, err := goListCommand.Output() if err != nil { return fmt.Errorf("Failed to get output of go list -m -json all: %v", err) @@ -488,3 +494,48 @@ func regenerateGodepsFromGoMod() error { } return nil } + +func checkoutBranchTagCommit(r *gogit.Repository, bh plumbing.Hash, dependentRepos []string) *gogit.Worktree { + fmt.Printf("Checking that dependencies point to the actual tags in %s.\n", strings.Join(dependentRepos, ", ")) + wt, err := r.Worktree() + if err != nil { + glog.Fatalf("Failed to get working tree: %v", err) + } + + fmt.Printf("Checking out branch tag commit %s.\n", bh.String()) + if err := wt.Checkout(&gogit.CheckoutOptions{Hash: bh}); err != nil { + glog.Fatalf("Failed to checkout %v: %v", bh, err) + } + return wt +} + +func updateGoModAndGodeps(tag string, dependentRepos []string, generateGodeps, publishSemverTags bool) bool { + fmt.Printf("Updating go.mod and go.sum to point to %s tag.\n", tag) + changed, err := updateGomodWithTaggedDependencies(tag, dependentRepos, publishSemverTags) + if err != nil { + glog.Fatalf("Failed to update go.mod and go.sum for tag %s: %v", tag, err) + } + + if changed && generateGodeps { + fmt.Printf("Regenerating Godeps.json from go.mod.\n") + if err := regenerateGodepsFromGoMod(); err != nil { + glog.Fatalf("Failed to regenerate Godeps.json from go.mod: %v", err) + } + } + return changed +} + +func createCommitToFixDeps(wt *gogit.Worktree, tag string) plumbing.Hash { + fmt.Printf("Adding extra commit to update dependencies to %s tag.\n", tag) + publishingBotNow := publishingBot + publishingBotNow.When = time.Now() + bh, err := wt.Commit(fmt.Sprintf("Update dependencies to %s tag", tag), &gogit.CommitOptions{ + All: true, + Author: &publishingBotNow, + Committer: &publishingBotNow, + }) + if err != nil { + glog.Fatalf("Failed to commit changes to update dependencies to %s tag: %v", tag, err) + } + return bh +}