Skip to content

Commit

Permalink
sync-tags: add support for v0.x.y tags
Browse files Browse the repository at this point in the history 
Implements changes to the KEP in kubernetes/enhancements#1350.

This commit updates sync-tags to publish `v0.x.y` tags for corresponding
`v1.x.y` Kubernetes tag. The old `kubernetes-1.x.y` tags are still also
published. Both `kubernetes-1.x.y` and `v0.x.y` point to the same sha.

For `v0.x.y`:

- At this tag, `go.mod` uses `v0.x.y` tags for dependencies.
- When this tag is manually specified in a `go.mod` file, it will be
retained as `v0.x.y`.

For `kubernetes-1.x.y`:

- At this tag, `go.mod` uses `v0.x.y` tags for dependencies. Note: this
differs from the previous way, where pseudoversions were used.
- When this tag is manually specified in a `go.mod` file, go will resolve
it to the appropriate pseudoversion.
  • Loading branch information
@nikhita
nikhita committed Dec 5, 2019
1 parent dc65f29 commit 542b65e
Show file tree
Hide file tree
Showing 3 changed files with 127 additions and 72 deletions.
1 change: 1 addition & 0 deletions artifacts/scripts/construct.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ if [ -z "${SKIP_TAGS}" ]; then
--dependencies "${DEPS}" \
--mapping-output-file "../tag-${REPO}-{{.Tag}}-mapping" \
--generate-godeps=${PUBLISHER_BOT_GENERATE_GODEPS:-false} \
--publish-semver-tags \
-alsologtostderr \
"${EXTRA_ARGS[@]-}"
if [ "${LAST_HEAD}" != "$(git rev-parse ${LAST_BRANCH})" ]; then
Expand Down
56 changes: 30 additions & 26 deletions cmd/sync-tags/gomod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import (

// updateGomodWithTaggedDependencies gets the dependencies at the given tag and fills go.mod and go.sum.
// If anything is changed, it commits the changes. Returns true if go.mod changed.
func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, error) {
func updateGomodWithTaggedDependencies(tag string, depsRepo []string, semverTag bool) (bool, error) {
found := map[string]bool{}
changed := false

Expand All @@ -59,10 +59,14 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err
return changed, fmt.Errorf("failed to get tag %s for %q: %v", tag, depPkg, err)
}
rev := commit.String()
pseudoVersion := fmt.Sprintf("v0.0.0-%s-%s", commitTime.UTC().Format("20060102150405"), rev[:12])
pseudoVersionOrTag := fmt.Sprintf("v0.0.0-%s-%s", commitTime.UTC().Format("20060102150405"), rev[:12])

// in case the pseudoVersion has not changed, running go mod download will help
// in avoiding packaging it up if the pseudoVersion has been published already
if semverTag {
pseudoVersionOrTag = tag
}

// in case the pseudoVersion/tag has not changed, running go mod download will help
// in avoiding packaging it up if the pseudoVersion/tag has been published already
downloadCommand := exec.Command("go", "mod", "download")
downloadCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
downloadCommand.Stdout = os.Stdout
Expand All @@ -71,46 +75,46 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err
return changed, fmt.Errorf("error running go mod download for %s: %v", depPkg, err)
}

// check if we have the pseudoVersion published already. if we don't, package it up
// check if we have the pseudoVersion/tag published already. if we don't, package it up
// and save to local mod download cache.
if err := packageDepToGoModCache(depPath, depPkg, rev, pseudoVersion, commitTime); err != nil {
if err := packageDepToGoModCache(depPath, depPkg, rev, pseudoVersionOrTag, commitTime); err != nil {
return changed, fmt.Errorf("failed to package %s dependency: %v", depPkg, err)
}

requireCommand := exec.Command("go", "mod", "edit", "-fmt", "-require", fmt.Sprintf("%s@%s", depPkg, pseudoVersion))
requireCommand := exec.Command("go", "mod", "edit", "-fmt", "-require", fmt.Sprintf("%s@%s", depPkg, pseudoVersionOrTag))
requireCommand.Env = append(os.Environ(), "GO111MODULE=on")
requireCommand.Stdout = os.Stdout
requireCommand.Stderr = os.Stderr
if err := requireCommand.Run(); err != nil {
return changed, fmt.Errorf("unable to pin %s in the require section of go.mod to %s: %v", depPkg, pseudoVersion, err)
return changed, fmt.Errorf("unable to pin %s in the require section of go.mod to %s: %v", depPkg, pseudoVersionOrTag, err)
}

replaceCommand := exec.Command("go", "mod", "edit", "-fmt", "-replace", fmt.Sprintf("%s=%s@%s", depPkg, depPkg, pseudoVersion))
replaceCommand := exec.Command("go", "mod", "edit", "-fmt", "-replace", fmt.Sprintf("%s=%s@%s", depPkg, depPkg, pseudoVersionOrTag))
replaceCommand.Env = append(os.Environ(), "GO111MODULE=on")
replaceCommand.Stdout = os.Stdout
replaceCommand.Stderr = os.Stderr
if err := replaceCommand.Run(); err != nil {
return changed, fmt.Errorf("unable to pin %s in the replace section of go.mod to %s: %v", depPkg, pseudoVersion, err)
return changed, fmt.Errorf("unable to pin %s in the replace section of go.mod to %s: %v", depPkg, pseudoVersionOrTag, err)
}

downloadCommand2 := exec.Command("go", "mod", "download")
downloadCommand2.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
downloadCommand2.Stdout = os.Stdout
downloadCommand2.Stderr = os.Stderr
if err := downloadCommand2.Run(); err != nil {
return changed, fmt.Errorf("error running go mod download for pseudo-version %s for %s: %v", pseudoVersion, depPkg, err)
return changed, fmt.Errorf("error running go mod download for pseudo-version %s for %s: %v", pseudoVersionOrTag, depPkg, err)
}

tidyCommand := exec.Command("go", "mod", "tidy")
tidyCommand.Env = append(os.Environ(), "GO111MODULE=on", "GOPOXY=file://${GOPATH}/pkg/mod/cache/download")
tidyCommand.Env = append(os.Environ(), "GO111MODULE=on")
tidyCommand.Stdout = os.Stdout
tidyCommand.Stderr = os.Stderr
if err := tidyCommand.Run(); err != nil {
return changed, fmt.Errorf("unable to run go mod tidy for %s at %s: %v", depPkg, rev, err)
}

found[dep] = true
fmt.Printf("Bumping %s in go.mod to %s\n.", depPkg, rev)
fmt.Printf("Bumping %s in go.mod to %s.\n", depPkg, rev)
changed = true
}

Expand Down Expand Up @@ -146,18 +150,18 @@ type ModuleInfo struct {
Time string
}

func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commitTime time.Time) error {
func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersionOrTag string, commitTime time.Time) error {
cacheDir := fmt.Sprintf("%s/pkg/mod/cache/download/%s/@v", os.Getenv("GOPATH"), depPkg)
goModFile := fmt.Sprintf("%s/%s.mod", cacheDir, pseudoVersion)
goModFile := fmt.Sprintf("%s/%s.mod", cacheDir, pseudoVersionOrTag)

if _, err := os.Stat(goModFile); err == nil {
fmt.Printf("Pseudo version %s for %s is already packaged up.\n", pseudoVersion, depPkg)
fmt.Printf("%s for %s is already packaged up.\n", pseudoVersionOrTag, depPkg)
return nil
} else if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("Could not check if %s exists: %v", goModFile, err)
}

fmt.Printf("Packaging up pseudo version %s for %s into go mod cache.\n", pseudoVersion, depPkg)
fmt.Printf("Packaging up %s for %s into go mod cache.\n", pseudoVersionOrTag, depPkg)

// create the cache if it doesn't exist
if err := os.MkdirAll(filepath.Dir(goModFile), os.FileMode(755)); err != nil {
Expand All @@ -173,14 +177,14 @@ func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commi
return fmt.Errorf("failed to checkout %s at %s: %v", depPkg, commit, err)
}

// copy go.mod to pseudoVersion.mod in the cache dir
// copy go.mod to the cache dir
if err := copyFile(fmt.Sprintf("%s/go.mod", depPath), goModFile); err != nil {
return fmt.Errorf("unable to copy %s file to %s to gomod cache for %s: %v", fmt.Sprintf("%s/go.mod", depPath), goModFile, depPkg, err)
}

// create pseudoVersion.info file in the cache dir
// create info file in the cache dir
moduleInfo := ModuleInfo{
Version: pseudoVersion,
Version: pseudoVersionOrTag,
Name: commit,
Short: commit[:12],
Time: commitTime.UTC().Format("2006-01-02T15:04:05Z"),
Expand All @@ -190,17 +194,17 @@ func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commi
if err != nil {
return fmt.Errorf("error marshaling .info file for %s: %v", depPkg, err)
}
if err := ioutil.WriteFile(fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersion), moduleFile, 0644); err != nil {
return fmt.Errorf("failed to write %s file for %s: %v", fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersion), depPkg, err)
if err := ioutil.WriteFile(fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), moduleFile, 0644); err != nil {
return fmt.Errorf("failed to write %s file for %s: %v", fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), depPkg, err)
}

// create the pseudoVersion.zip file in the cache dir. This zip file has the same hash
// create the zip file in the cache dir. This zip file has the same hash
// as of the zip file that would have been created by go mod download.
zipCommand := exec.Command("/gomod-zip", "--package-name", depPkg, "--pseudo-version", pseudoVersion)
zipCommand := exec.Command("/gomod-zip", "--package-name", depPkg, "--pseudo-version", pseudoVersionOrTag)
zipCommand.Stdout = os.Stdout
zipCommand.Stderr = os.Stderr
if err := zipCommand.Run(); err != nil {
return fmt.Errorf("failed to run gomod-zip for %s at %s: %v", depPkg, pseudoVersion, err)
return fmt.Errorf("failed to run gomod-zip for %s at %s: %v", depPkg, pseudoVersionOrTag, err)
}

// append the pseudoVersion to the list file in the cache dir
Expand All @@ -210,7 +214,7 @@ func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersion string, commi
}
defer listFile.Close()

if _, err := listFile.WriteString(fmt.Sprintf("%s\n", pseudoVersion)); err != nil {
if _, err := listFile.WriteString(fmt.Sprintf("%s\n", pseudoVersionOrTag)); err != nil {
return fmt.Errorf("unable to write to list file in %s: %v", cacheDir, err)
}

Expand Down
142 changes: 96 additions & 46 deletions cmd/sync-tags/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ func main() {
skipFetch := flag.Bool("skip-fetch", false, "skip fetching tags")
generateGodeps := flag.Bool("generate-godeps", false, "regenerate Godeps.json from go.mod")
mappingOutputFile := flag.String("mapping-output-file", "", "a file name to write the source->dest hash mapping to ({{.Tag}} is substituted with the tag name, {{.Branch}} with the local branch name)")
publishSemverTags := flag.Bool("publish-semver-tags", false, "publish v0.x.y tag at destination repo for v1.x.y tag at the source repo")

flag.Usage = Usage
flag.Parse()
Expand Down Expand Up @@ -186,6 +187,7 @@ func main() {
if *prefix != "" {
bName = *prefix + name[1:] // remove the v
}
semverTag := name[:1] + "0" + name[2:] // replace v1.x.y with v0.x.y

// ignore non-annotated tags
tag, err := r.TagObject(kh)
Expand All @@ -199,15 +201,20 @@ func main() {
continue
}

// skip if it already exists in origin
if _, found := bTagCommits[bName]; found {
// if semver and non-semver tags need to be published, skip if both tags already exist
// if only non-semver tags need to be published, skip if the non-semver tag already exists
_, nonSemverTagExists := bTagCommits[bName]
_, semverTagExists := bTagCommits[semverTag]
if *publishSemverTags && semverTagExists && nonSemverTagExists {
continue
} else if nonSemverTagExists {
continue
}

// do not override tags (we build master first, i.e. the x.y.z-alpha.0 tag on master will not be created for feature branches)
if tagExists(r, bName) {
continue
}
// if tagExists(r, bName) {
// continue
// }

// lazily compute kube commit map
if sourceCommitsToDstCommits == nil {
Expand Down Expand Up @@ -258,71 +265,69 @@ func main() {
// update go.mod or Godeps.json to point to actual tagged version in the dependencies. This version might differ
// from the one currently in go.mod or Godeps.json because the other repo could have gotten more commit for this
// tag, but this repo didn't. Compare https://github.com/kubernetes/publishing-bot/issues/12 for details.
var changed, goModExists bool
_, err = os.Stat("go.mod")
if err == nil {
goModExists = true
}

if len(dependentRepos) > 0 {
fmt.Printf("Checking that dependencies point to the actual tags in %s.\n", strings.Join(dependentRepos, ", "))
wt, err := r.Worktree()
if err != nil {
glog.Fatalf("Failed to get working tree: %v", err)
}
fmt.Printf("Checking out branch tag commit %s.\n", bh.String())
if err := wt.Checkout(&gogit.CheckoutOptions{Hash: bh}); err != nil {
glog.Fatalf("Failed to checkout %v: %v", bh, err)
}
wt := checkoutBranchTagCommit(r, bh, dependentRepos)

// if go.mod exists, fix only go.mod and generate Godeps.json from it later
// if it doesn't exist, check if Godeps.json exists, and update it
var changed, goModChanged bool
_, err = os.Stat("go.mod")
if os.IsNotExist(err) {
if !goModExists {
if _, err2 := os.Stat("Godeps/Godeps.json"); err2 == nil {
fmt.Printf("Updating Godeps.json to point to %s tag.\n", bName)
changed, err = updateGodepsJsonWithTaggedDependencies(bName, dependentRepos)
if err != nil {
glog.Fatalf("Failed to update Godeps.json for tag %s: %v", bName, err)
}
}
} else if err == nil {
fmt.Printf("Updating go.mod and go.sum to point to %s tag.\n", bName)
changed, err = updateGomodWithTaggedDependencies(bName, dependentRepos)
if err != nil {
glog.Fatalf("Failed to update go.mod and go.sum for tag %s: %v", bName, err)
}
goModChanged = true
}

if goModChanged && *generateGodeps {
fmt.Printf("Regenerating Godeps.json from go.mod.\n")
if err := regenerateGodepsFromGoMod(); err != nil {
glog.Fatalf("Failed to regenerate Godeps.json from go.mod: %v", err)
} else {
if *publishSemverTags {
changed = updateGoModAndGodeps(semverTag, dependentRepos, *generateGodeps, true)
} else {
changed = updateGoModAndGodeps(bName, dependentRepos, *generateGodeps, false)
}
}

if changed {
fmt.Printf("Adding extra commit fixing dependencies to point to %s tags.\n", bName)
publishingBotNow := publishingBot
publishingBotNow.When = time.Now()
bh, err = wt.Commit(fmt.Sprintf("Fix dependencies to point to %s tag", bName), &gogit.CommitOptions{
All: true,
Author: &publishingBotNow,
Committer: &publishingBotNow,
})
if err != nil {
glog.Fatalf("Failed to commit changes to fix dependencies to point to %s tag: %v", bName, err)
if *publishSemverTags {
bh = createCommitToFixDeps(wt, semverTag)
} else {
bh = createCommitToFixDeps(wt, bName)
}
}
}

// create prefixed annotated tag
fmt.Printf("Tagging %v as %q.\n", bh, bName)
err = createAnnotatedTag(bh, bName, tag.Tagger.When, dedent.Dedent(fmt.Sprintf(`
// create semver annotated tag
if *publishSemverTags && !semverTagExists {
fmt.Printf("Tagging %v as %q.\n", bh, semverTag)
err = createAnnotatedTag(bh, semverTag, tag.Tagger.When, dedent.Dedent(fmt.Sprintf(`
Kubernetes release %s
Based on https://github.com/kubernetes/kubernetes/releases/tag/%s
`, name, name)))
if err != nil {
glog.Fatalf("Failed to create tag %q: %v", bName, err)
if err != nil {
glog.Fatalf("Failed to create tag %q: %v", semverTag, err)
}
createdTags = append(createdTags, semverTag)
}

// create non-semver prefixed annotated tag
if !nonSemverTagExists {
fmt.Printf("Tagging %v as %q.\n", bh, bName)
err = createAnnotatedTag(bh, bName, tag.Tagger.When, dedent.Dedent(fmt.Sprintf(`
Kubernetes release %s
Based on https://github.com/kubernetes/kubernetes/releases/tag/%s
`, name, name)))
if err != nil {
glog.Fatalf("Failed to create tag %q: %v", bName, err)
}
createdTags = append(createdTags, bName)
}
createdTags = append(createdTags, bName)
}

// write push command for new tags
Expand Down Expand Up @@ -488,3 +493,48 @@ func regenerateGodepsFromGoMod() error {
}
return nil
}

func checkoutBranchTagCommit(r *gogit.Repository, bh plumbing.Hash, dependentRepos []string) *gogit.Worktree {
fmt.Printf("Checking that dependencies point to the actual tags in %s.\n", strings.Join(dependentRepos, ", "))
wt, err := r.Worktree()
if err != nil {
glog.Fatalf("Failed to get working tree: %v", err)
}

fmt.Printf("Checking out branch tag commit %s.\n", bh.String())
if err := wt.Checkout(&gogit.CheckoutOptions{Hash: bh}); err != nil {
glog.Fatalf("Failed to checkout %v: %v", bh, err)
}
return wt
}

func updateGoModAndGodeps(tag string, dependentRepos []string, generateGodeps, publishSemverTags bool) bool {
fmt.Printf("Updating go.mod and go.sum to point to %s tag.\n", tag)
changed, err := updateGomodWithTaggedDependencies(tag, dependentRepos, publishSemverTags)
if err != nil {
glog.Fatalf("Failed to update go.mod and go.sum for tag %s: %v", tag, err)
}

if changed && generateGodeps {
fmt.Printf("Regenerating Godeps.json from go.mod.\n")
if err := regenerateGodepsFromGoMod(); err != nil {
glog.Fatalf("Failed to regenerate Godeps.json from go.mod: %v", err)
}
}
return changed
}

func createCommitToFixDeps(wt *gogit.Worktree, tag string) plumbing.Hash {
fmt.Printf("Adding extra commit fixing dependencies to point to %s tags.\n", tag)
publishingBotNow := publishingBot
publishingBotNow.When = time.Now()
bh, err := wt.Commit(fmt.Sprintf("Fix dependencies to point to %s tag", tag), &gogit.CommitOptions{
All: true,
Author: &publishingBotNow,
Committer: &publishingBotNow,
})
if err != nil {
glog.Fatalf("Failed to commit changes to fix dependencies to point to %s tag: %v", tag, err)
}
return bh
}

0 comments on commit 542b65e

Please sign in to comment.