From 47478c32edf563cd77553efd746c48001b0ade58 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 8 Mar 2023 17:24:51 +0200 Subject: [PATCH 01/34] initial addition of the go_upgrade tool to detect and bump the golang version Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 35 +++ go.mod | 1 + go.sum | 2 + go/tools/go-upgrade/go_upgrade.go | 229 ++++++++++++++++++++ 4 files changed, 267 insertions(+) create mode 100644 .github/workflows/update_golang_version.yml create mode 100644 go/tools/go-upgrade/go_upgrade.go diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml new file mode 100644 index 00000000000..cfe8abdcfd0 --- /dev/null +++ b/.github/workflows/update_golang_version.yml @@ -0,0 +1,35 @@ +name: Update Golang Version + +on: + # Triggers the workflow every week + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +jobs: + update_golang_version: + name: Update Golang Version + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.20.2 + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Detect new version and update codebase + run: + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v4 + with: + commit-message: "upgrade vitess dependency to latest" + signoff: true + delete-branch: true + title: "Upgrade Vitess Dependency to Latest" + body: "Weekly Vitess dependency upgrade running on a cron schedule" + reviewers: | + frouioui + GuptaManan100 diff --git a/go.mod b/go.mod index 1c82c9f0942..aa8f87bef0c 100644 --- a/go.mod +++ b/go.mod @@ -154,6 +154,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v1.4.0 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index e47c7dda131..43b1be37384 100644 --- a/go.sum +++ b/go.sum @@ -460,6 +460,8 @@ github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2I github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go new file mode 100644 index 00000000000..abbf34302d7 --- /dev/null +++ b/go/tools/go-upgrade/go_upgrade.go @@ -0,0 +1,229 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path" + "strings" + + "github.com/hashicorp/go-version" +) + +const ( + goDevAPI = "https://go.dev/dl/?mode=json" +) + +type latestGolangRelease struct { + Version string `json:"version"` + Stable bool `json:"stable"` +} + +func main() { + failIfNoUpdate := false + allowMajorUpgrade := false + + currentVersion, err := currentGolangVersion() + if err != nil { + log.Fatal(err) + } + + availableVersions, err := getLatestStableGolangReleases() + if err != nil { + log.Fatal(err) + } + + upgradeTo := chooseNewVersion(currentVersion, availableVersions, allowMajorUpgrade) + if upgradeTo == nil { + if failIfNoUpdate { + os.Exit(1) + } + return + } + + err = replaceGoVersionInCodebase(currentVersion, upgradeTo) + if err != nil { + log.Fatal(err) + } + +} + +// currentGolangVersion gets the running version of Golang in Vitess +// and returns it as a *version.Version. +// +// The file `./build.env` describes which version of Golang is expected by Vitess. +// We use this file to detect the current Golang version of our codebase. +// The file contains `goversion_min 1.20.1`, we will grep `goversion_min` to finally find +// the precise golang version we're using. +func currentGolangVersion() (*version.Version, error) { + contentRaw, err := os.ReadFile("build.env") + if err != nil { + return nil, err + } + content := string(contentRaw) + idxBegin := strings.Index(content, "goversion_min") + len("goversion_min") + 1 + idxFinish := strings.Index(content[idxBegin:], "||") + idxBegin + versionStr := strings.TrimSpace(content[idxBegin:idxFinish]) + return version.NewVersion(versionStr) +} + +// getLatestStableGolangReleases fetches the latest stable releases of Golang from +// the official website using the goDevAPI URL. +// Once fetched, the releases are returned as version.Collection. +func getLatestStableGolangReleases() (version.Collection, error) { + resp, err := http.Get(goDevAPI) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var latestGoReleases []latestGolangRelease + err = json.Unmarshal(body, &latestGoReleases) + if err != nil { + return nil, err + } + + var versions version.Collection + for _, release := range latestGoReleases { + if !release.Stable { + continue + } + if !strings.HasPrefix(release.Version, "go") { + return nil, fmt.Errorf("golang version malformatted: %s", release.Version) + } + newVersion, err := version.NewVersion(release.Version[2:]) + if err != nil { + return nil, err + } + versions = append(versions, newVersion) + } + return versions, nil +} + +// chooseNewVersion decides what will be the next version we're going to use in our codebase. +// Given the current Golang version, the available latest versions and whether we allow major upgrade or not, +// chooseNewVersion will return either the new version or nil if we cannot/don't need to upgrade. +func chooseNewVersion(curVersion *version.Version, latestVersions version.Collection, allowMajorUpgrade bool) *version.Version { + selectedVersion := curVersion + for _, latestVersion := range latestVersions { + if !allowMajorUpgrade && latestVersion.Segments()[0] != selectedVersion.Segments()[0] { + continue + } + if latestVersion.GreaterThan(curVersion) { + selectedVersion = latestVersion + } + } + // No change detected, return nil meaning that we do not want to have a new Golang version. + if selectedVersion.Equal(curVersion) { + return nil + } + return selectedVersion +} + +// replaceGoVersionInCodebase goes through all the files in the codebase where the +// Golang version must be updated +func replaceGoVersionInCodebase(old, new *version.Version) error { + filesToChange, err := getListOfFilesWhereGoVersionMustBeUpdated() + if err != nil { + return err + } + err = writeNewGolangVersionToFiles(old, new, filesToChange) + if err != nil { + return err + } + return nil +} + +// writeNewGolangVersionToFiles goes through all the given file and call +// writeNewGolangVersionToSingleFile to replace one by one the files with +// the new Golang version. +func writeNewGolangVersionToFiles(old, new *version.Version, filesToChange []string) error { + for _, fileToChange := range filesToChange { + err := writeNewGolangVersionToSingleFile(old, new, fileToChange) + if err != nil { + return err + } + } + return nil +} + +// writeNewGolangVersionToSingleFile replaces old with new in the given file. +func writeNewGolangVersionToSingleFile(old *version.Version, new *version.Version, fileToChange string) error { + f, err := os.OpenFile(fileToChange, os.O_RDWR, 0600) + if err != nil { + return err + } + defer f.Close() + + content, err := io.ReadAll(f) + if err != nil { + return err + } + contentStr := string(content) + + newContent := strings.ReplaceAll(contentStr, old.String(), new.String()) + + _, err = f.WriteAt([]byte(newContent), 0) + if err != nil { + return err + } + return nil +} + +// getListOfFilesWhereGoVersionMustBeUpdated returns the list of all the files where +// the Golang version must be updated, excluding go.mod which uses a different format. +func getListOfFilesWhereGoVersionMustBeUpdated() ([]string, error) { + pathsToExplore := []string{ + "./.github/workflows", + "./test/templates", + "./build.env", + "./docker/bootstrap/Dockerfile.common", + } + + var filesToChange []string + for _, pathToExplore := range pathsToExplore { + stat, err := os.Stat(pathToExplore) + if err != nil { + return nil, err + } + if stat.IsDir() { + dirEntries, err := os.ReadDir(pathToExplore) + if err != nil { + return nil, err + } + for _, entry := range dirEntries { + if entry.IsDir() { + continue + } + filesToChange = append(filesToChange, path.Join(pathToExplore, entry.Name())) + } + } else { + filesToChange = append(filesToChange, pathToExplore) + } + } + return filesToChange, nil +} From 69b49cd1472babf6a972f25517e2fbb57e305c19 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 8 Mar 2023 17:30:23 +0200 Subject: [PATCH 02/34] update go.mod automatically Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 36 ++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index abbf34302d7..379b1041edc 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -155,6 +155,13 @@ func replaceGoVersionInCodebase(old, new *version.Version) error { if err != nil { return err } + + if old.Segments()[0] != new.Segments()[0] { + err = writeNewMajorGolangVersionToGoMod(old, new) + if err != nil { + return err + } + } return nil } @@ -172,7 +179,7 @@ func writeNewGolangVersionToFiles(old, new *version.Version, filesToChange []str } // writeNewGolangVersionToSingleFile replaces old with new in the given file. -func writeNewGolangVersionToSingleFile(old *version.Version, new *version.Version, fileToChange string) error { +func writeNewGolangVersionToSingleFile(old, new *version.Version, fileToChange string) error { f, err := os.OpenFile(fileToChange, os.O_RDWR, 0600) if err != nil { return err @@ -194,6 +201,33 @@ func writeNewGolangVersionToSingleFile(old *version.Version, new *version.Versio return nil } +// writeNewMajorGolangVersionToGoMod bumps the major version of Golang found in the go.mod file. +func writeNewMajorGolangVersionToGoMod(old, new *version.Version) error { + f, err := os.OpenFile("./go.mod", os.O_RDWR, 0600) + if err != nil { + return err + } + defer f.Close() + + content, err := io.ReadAll(f) + if err != nil { + return err + } + contentStr := string(content) + + newContent := strings.ReplaceAll( + contentStr, + fmt.Sprintf("%d.%d", old.Segments()[0], old.Segments()[1]), + fmt.Sprintf("%d.%d", new.Segments()[0], new.Segments()[1]), + ) + + _, err = f.WriteAt([]byte(newContent), 0) + if err != nil { + return err + } + return nil +} + // getListOfFilesWhereGoVersionMustBeUpdated returns the list of all the files where // the Golang version must be updated, excluding go.mod which uses a different format. func getListOfFilesWhereGoVersionMustBeUpdated() ([]string, error) { From 67ecb1e41af52baf8c63029a0d44b698da7221dc Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 8 Mar 2023 17:43:08 +0200 Subject: [PATCH 03/34] fix isSameMajor version issue Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 379b1041edc..d2a11cc67a1 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -40,7 +40,7 @@ type latestGolangRelease struct { func main() { failIfNoUpdate := false - allowMajorUpgrade := false + allowMajorUpgrade := true currentVersion, err := currentGolangVersion() if err != nil { @@ -130,10 +130,10 @@ func getLatestStableGolangReleases() (version.Collection, error) { func chooseNewVersion(curVersion *version.Version, latestVersions version.Collection, allowMajorUpgrade bool) *version.Version { selectedVersion := curVersion for _, latestVersion := range latestVersions { - if !allowMajorUpgrade && latestVersion.Segments()[0] != selectedVersion.Segments()[0] { + if !allowMajorUpgrade && !isSameMajorVersion(latestVersion, selectedVersion) { continue } - if latestVersion.GreaterThan(curVersion) { + if latestVersion.GreaterThan(selectedVersion) { selectedVersion = latestVersion } } @@ -156,7 +156,7 @@ func replaceGoVersionInCodebase(old, new *version.Version) error { return err } - if old.Segments()[0] != new.Segments()[0] { + if !isSameMajorVersion(old, new) { err = writeNewMajorGolangVersionToGoMod(old, new) if err != nil { return err @@ -261,3 +261,7 @@ func getListOfFilesWhereGoVersionMustBeUpdated() ([]string, error) { } return filesToChange, nil } + +func isSameMajorVersion(a, b *version.Version) bool { + return a.Segments()[1] == b.Segments()[1] +} From 48605c9aab58124107bf9f0d64fbe225823dc210 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 8 Mar 2023 19:42:07 +0200 Subject: [PATCH 04/34] update the bootstrap version automatically Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 2 +- go/tools/go-upgrade/go_upgrade.go | 174 +++++++++++++------- 2 files changed, 111 insertions(+), 65 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index cfe8abdcfd0..00a06a6aaa9 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -1,7 +1,7 @@ name: Update Golang Version on: - # Triggers the workflow every week + # Triggers the workflow every day schedule: - cron: "0 0 * * *" workflow_dispatch: diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index d2a11cc67a1..ae05623c317 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -24,6 +24,7 @@ import ( "net/http" "os" "path" + "strconv" "strings" "github.com/hashicorp/go-version" @@ -40,7 +41,8 @@ type latestGolangRelease struct { func main() { failIfNoUpdate := false - allowMajorUpgrade := true + allowMajorUpgrade := false + isMainBranch := true currentVersion, err := currentGolangVersion() if err != nil { @@ -65,6 +67,21 @@ func main() { log.Fatal(err) } + currentBootstrapVersionF, err := currentBootstrapVersion() + if err != nil { + log.Fatal(err) + } + nextBootstrapVersionF := currentBootstrapVersionF + if isMainBranch { + nextBootstrapVersionF += 1 + } else { + nextBootstrapVersionF += 0.1 + } + err = updateBootstrapVersionInCodebase(currentBootstrapVersionF, nextBootstrapVersionF, upgradeTo) + if err != nil { + log.Fatal(err) + } + } // currentGolangVersion gets the running version of Golang in Vitess @@ -86,6 +103,22 @@ func currentGolangVersion() (*version.Version, error) { return version.NewVersion(versionStr) } +func currentBootstrapVersion() (float64, error) { + contentRaw, err := os.ReadFile("Makefile") + if err != nil { + return 0, err + } + content := string(contentRaw) + idxBegin := strings.Index(content, "BOOTSTRAP_VERSION=") + len("BOOTSTRAP_VERSION=") + idxFinish := strings.IndexByte(content[idxBegin:], '\n') + idxBegin + versionStr := strings.TrimSpace(content[idxBegin:idxFinish]) + f, err := strconv.ParseFloat(versionStr, 64) + if err != nil { + return 0, err + } + return f, nil +} + // getLatestStableGolangReleases fetches the latest stable releases of Golang from // the official website using the goDevAPI URL. // Once fetched, the releases are returned as version.Collection. @@ -147,30 +180,33 @@ func chooseNewVersion(curVersion *version.Version, latestVersions version.Collec // replaceGoVersionInCodebase goes through all the files in the codebase where the // Golang version must be updated func replaceGoVersionInCodebase(old, new *version.Version) error { - filesToChange, err := getListOfFilesWhereGoVersionMustBeUpdated() - if err != nil { - return err - } - err = writeNewGolangVersionToFiles(old, new, filesToChange) + filesToChange, err := getListOfFilesInPaths([]string{ + "./.github/workflows", + "./test/templates", + "./build.env", + "./docker/bootstrap/Dockerfile.common", + }) if err != nil { return err } - if !isSameMajorVersion(old, new) { - err = writeNewMajorGolangVersionToGoMod(old, new) + for _, fileToChange := range filesToChange { + err = replaceInFile( + []string{old.String()}, + []string{new.String()}, + fileToChange, + ) if err != nil { return err } } - return nil -} -// writeNewGolangVersionToFiles goes through all the given file and call -// writeNewGolangVersionToSingleFile to replace one by one the files with -// the new Golang version. -func writeNewGolangVersionToFiles(old, new *version.Version, filesToChange []string) error { - for _, fileToChange := range filesToChange { - err := writeNewGolangVersionToSingleFile(old, new, fileToChange) + if !isSameMajorVersion(old, new) { + err = replaceInFile( + []string{fmt.Sprintf("%d.%d", old.Segments()[0], old.Segments()[1])}, + []string{fmt.Sprintf("%d.%d", new.Segments()[0], new.Segments()[1])}, + "./go.mod", + ) if err != nil { return err } @@ -178,66 +214,51 @@ func writeNewGolangVersionToFiles(old, new *version.Version, filesToChange []str return nil } -// writeNewGolangVersionToSingleFile replaces old with new in the given file. -func writeNewGolangVersionToSingleFile(old, new *version.Version, fileToChange string) error { - f, err := os.OpenFile(fileToChange, os.O_RDWR, 0600) - if err != nil { - return err - } - defer f.Close() - - content, err := io.ReadAll(f) - if err != nil { - return err - } - contentStr := string(content) - - newContent := strings.ReplaceAll(contentStr, old.String(), new.String()) - - _, err = f.WriteAt([]byte(newContent), 0) - if err != nil { - return err - } - return nil -} - -// writeNewMajorGolangVersionToGoMod bumps the major version of Golang found in the go.mod file. -func writeNewMajorGolangVersionToGoMod(old, new *version.Version) error { - f, err := os.OpenFile("./go.mod", os.O_RDWR, 0600) +func updateBootstrapVersionInCodebase(old, new float64, newGoVersion *version.Version) error { + files, err := getListOfFilesInPaths([]string{ + "./docker/base", + "./docker/lite", + "./docker/local", + "./docker/vttestserver", + "./Makefile", + "./test/templates", + }) if err != nil { return err } - defer f.Close() - content, err := io.ReadAll(f) - if err != nil { - return err + const btv = "bootstrap_version=" + oldReplace := fmt.Sprintf("%s%-1g", btv, old) + newReplace := fmt.Sprintf("%s%-1g", btv, new) + for _, file := range files { + err = replaceInFile( + []string{oldReplace, strings.ToUpper(oldReplace)}, + []string{newReplace, strings.ToUpper(newReplace)}, + file, + ) + if err != nil { + return err + } } - contentStr := string(content) - newContent := strings.ReplaceAll( - contentStr, - fmt.Sprintf("%d.%d", old.Segments()[0], old.Segments()[1]), - fmt.Sprintf("%d.%d", new.Segments()[0], new.Segments()[1]), + oldReplace = fmt.Sprintf("\"bootstrap-version\", \"%-1g\"", old) + newReplace = fmt.Sprintf("\"bootstrap-version\", \"%-1g\"", new) + err = replaceInFile( + []string{oldReplace}, + []string{newReplace}, + "./test.go", ) - - _, err = f.WriteAt([]byte(newContent), 0) if err != nil { return err } return nil } -// getListOfFilesWhereGoVersionMustBeUpdated returns the list of all the files where -// the Golang version must be updated, excluding go.mod which uses a different format. -func getListOfFilesWhereGoVersionMustBeUpdated() ([]string, error) { - pathsToExplore := []string{ - "./.github/workflows", - "./test/templates", - "./build.env", - "./docker/bootstrap/Dockerfile.common", - } +func isSameMajorVersion(a, b *version.Version) bool { + return a.Segments()[1] == b.Segments()[1] +} +func getListOfFilesInPaths(pathsToExplore []string) ([]string, error) { var filesToChange []string for _, pathToExplore := range pathsToExplore { stat, err := os.Stat(pathToExplore) @@ -262,6 +283,31 @@ func getListOfFilesWhereGoVersionMustBeUpdated() ([]string, error) { return filesToChange, nil } -func isSameMajorVersion(a, b *version.Version) bool { - return a.Segments()[1] == b.Segments()[1] +// replaceInFile replaces old with new in the given file. +func replaceInFile(old, new []string, fileToChange string) error { + if len(old) != len(new) { + panic("old and new should be of the same length") + } + + f, err := os.OpenFile(fileToChange, os.O_RDWR, 0600) + if err != nil { + return err + } + defer f.Close() + + content, err := io.ReadAll(f) + if err != nil { + return err + } + contentStr := string(content) + + for i := range old { + contentStr = strings.ReplaceAll(contentStr, old[i], new[i]) + } + + _, err = f.WriteAt([]byte(contentStr), 0) + if err != nil { + return err + } + return nil } From 5cfb9b08ab242951279792d4a3073bfc45a32a35 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 8 Mar 2023 19:52:40 +0200 Subject: [PATCH 05/34] update the bootstrap docker image changelog Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 39 +++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index ae05623c317..e33f80474f2 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -26,6 +26,7 @@ import ( "path" "strconv" "strings" + "time" "github.com/hashicorp/go-version" ) @@ -62,10 +63,10 @@ func main() { return } - err = replaceGoVersionInCodebase(currentVersion, upgradeTo) - if err != nil { - log.Fatal(err) - } + // err = replaceGoVersionInCodebase(currentVersion, upgradeTo) + // if err != nil { + // log.Fatal(err) + // } currentBootstrapVersionF, err := currentBootstrapVersion() if err != nil { @@ -251,6 +252,36 @@ func updateBootstrapVersionInCodebase(old, new float64, newGoVersion *version.Ve if err != nil { return err } + + err = updateBootstrapChangelog(new, newGoVersion) + if err != nil { + return err + } + + return nil +} + +func updateBootstrapChangelog(new float64, goVersion *version.Version) error { + file, err := os.OpenFile("./docker/bootstrap/CHANGELOG.md", os.O_RDWR, 0600) + if err != nil { + return err + } + defer file.Close() + + s, err := file.Stat() + if err != nil { + return err + } + newContent := fmt.Sprintf(` + +## [%-1g] - %s +### Changes +- Update build to golang %s`, new, time.Now().Format(time.DateOnly), goVersion.String()) + + _, err = file.WriteAt([]byte(newContent), s.Size()) + if err != nil { + return err + } return nil } From 365382603813cec9c526bd2be15c446c727105c7 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 8 Mar 2023 20:20:30 +0200 Subject: [PATCH 06/34] uncomment code Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index e33f80474f2..7c25674966b 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -63,10 +63,10 @@ func main() { return } - // err = replaceGoVersionInCodebase(currentVersion, upgradeTo) - // if err != nil { - // log.Fatal(err) - // } + err = replaceGoVersionInCodebase(currentVersion, upgradeTo) + if err != nil { + log.Fatal(err) + } currentBootstrapVersionF, err := currentBootstrapVersion() if err != nil { From 89ec9adb603dcc5e5721c0e0cc52690e34d0777d Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 8 Mar 2023 20:35:20 +0200 Subject: [PATCH 07/34] addition of workflow Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 23 ++++++++++++++++----- go/tools/go-upgrade/go_upgrade.go | 4 ---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 00a06a6aaa9..018ba297909 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -5,6 +5,7 @@ on: schedule: - cron: "0 0 * * *" workflow_dispatch: + pull_request: jobs: update_golang_version: @@ -20,16 +21,28 @@ jobs: uses: actions/checkout@v3 - name: Detect new version and update codebase - run: + id: detect-and-update + run: | + if [ ${{github.ref}} == "main" ]; then + go run ./go/tools/go-upgrade/go_upgrade.go --main --allow-major-upgrade + else + go run ./go/tools/go-upgrade/go_upgrade.go + fi + + output=$(git status -s) + if [ -z "${output}" ]; then + exit 0 + fi + echo "create-pr=true" >> $GITHUB_OUTPUT - name: Create Pull Request + if: steps.detect-and-update.outputs.create-pr == 'true' uses: peter-evans/create-pull-request@v4 with: - commit-message: "upgrade vitess dependency to latest" + commit-message: "Upgrade the Golang version" signoff: true delete-branch: true - title: "Upgrade Vitess Dependency to Latest" - body: "Weekly Vitess dependency upgrade running on a cron schedule" + title: "Bump the Golang version" + body: "test" reviewers: | frouioui - GuptaManan100 diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 7c25674966b..bba0251cafa 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -41,7 +41,6 @@ type latestGolangRelease struct { } func main() { - failIfNoUpdate := false allowMajorUpgrade := false isMainBranch := true @@ -57,9 +56,6 @@ func main() { upgradeTo := chooseNewVersion(currentVersion, availableVersions, allowMajorUpgrade) if upgradeTo == nil { - if failIfNoUpdate { - os.Exit(1) - } return } From 49986d4e48915a8e544ab6b8e346fe6aefe0a1a7 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 8 Mar 2023 20:39:03 +0200 Subject: [PATCH 08/34] addition of base to create-pull-request Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 018ba297909..be87cc7bee4 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -44,5 +44,6 @@ jobs: delete-branch: true title: "Bump the Golang version" body: "test" + base: "main" # TODO: remove, this is for test purposes reviewers: | frouioui From fea3561a6838d5691e82503c8d54fbc41acb12f2 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 09:52:32 +0200 Subject: [PATCH 09/34] test create PR alternative Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 30 +++++++++++++-------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index be87cc7bee4..83ba0076858 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -35,15 +35,23 @@ jobs: fi echo "create-pr=true" >> $GITHUB_OUTPUT - - name: Create Pull Request - if: steps.detect-and-update.outputs.create-pr == 'true' - uses: peter-evans/create-pull-request@v4 + - name: Create pull request + uses: devops-infra/action-pull-request@v0.5.5 with: - commit-message: "Upgrade the Golang version" - signoff: true - delete-branch: true - title: "Bump the Golang version" - body: "test" - base: "main" # TODO: remove, this is for test purposes - reviewers: | - frouioui + github_token: ${{ secrets.GITHUB_TOKEN }} + title: Automatic pull request + target_branch: main + reviewer: frouioui + +# - name: Create Pull Request +# if: steps.detect-and-update.outputs.create-pr == 'true' +# uses: peter-evans/create-pull-request@v4 +# with: +# commit-message: "Upgrade the Golang version" +# signoff: true +# delete-branch: true +# title: "Bump the Golang version" +# body: "test" +# base: "main" # TODO: remove, this is for test purposes +# reviewers: | +# frouioui From fc03b1d137135848004770f3ee29d9f7db9a583c Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 10:46:15 +0200 Subject: [PATCH 10/34] add flags and getter methods Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 2 + go/tools/go-upgrade/go_upgrade.go | 56 +++++++++++++++------ 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 83ba0076858..92ea4665beb 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -34,6 +34,8 @@ jobs: exit 0 fi echo "create-pr=true" >> $GITHUB_OUTPUT + git add --all + git commit -m - name: Create pull request uses: devops-infra/action-pull-request@v0.5.5 diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index bba0251cafa..2b15c09c457 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -18,6 +18,7 @@ package main import ( "encoding/json" + "flag" "fmt" "io" "log" @@ -41,32 +42,56 @@ type latestGolangRelease struct { } func main() { - allowMajorUpgrade := false - isMainBranch := true + noWorkflowUpdate := flag.Bool("no-workflow-update", false, "Whether or not the workflow files should be updated. Useful when using this script to auto-create PRs.") + allowMajorUpgrade := flag.Bool("allow-major-upgrade", false, "Defines if Golang major version upgrade are allowed.") + isMainBranch := flag.Bool("main", false, "Defines if the current branch is the main branch.") + flag.Parse() + + switch strings.ToLower(os.Args[1]) { + case "get_go_version": + currentVersion, err := currentGolangVersion() + if err != nil { + log.Fatal(err) + } + fmt.Println(currentVersion.String()) + case "get_bootstrap_version": + currentBootstrapVersionF, err := currentBootstrapVersion() + if err != nil { + log.Fatal(err) + } + fmt.Println(currentBootstrapVersionF) + default: + err := upgradePath(*allowMajorUpgrade, *noWorkflowUpdate, *isMainBranch) + if err != nil { + log.Fatal(err) + } + } +} +func upgradePath(allowMajorUpgrade, noWorkflowUpdate, isMainBranch bool) error { currentVersion, err := currentGolangVersion() if err != nil { - log.Fatal(err) + return err } availableVersions, err := getLatestStableGolangReleases() if err != nil { - log.Fatal(err) + return err } upgradeTo := chooseNewVersion(currentVersion, availableVersions, allowMajorUpgrade) if upgradeTo == nil { - return + return nil } - err = replaceGoVersionInCodebase(currentVersion, upgradeTo) + err = replaceGoVersionInCodebase(currentVersion, upgradeTo, noWorkflowUpdate) if err != nil { - log.Fatal(err) + return err } currentBootstrapVersionF, err := currentBootstrapVersion() if err != nil { - log.Fatal(err) + return err } nextBootstrapVersionF := currentBootstrapVersionF if isMainBranch { @@ -76,9 +101,9 @@ func main() { } err = updateBootstrapVersionInCodebase(currentBootstrapVersionF, nextBootstrapVersionF, upgradeTo) if err != nil { - log.Fatal(err) + return err } - + return nil } // currentGolangVersion gets the running version of Golang in Vitess @@ -176,13 +201,16 @@ func chooseNewVersion(curVersion *version.Version, latestVersions version.Collec // replaceGoVersionInCodebase goes through all the files in the codebase where the // Golang version must be updated -func replaceGoVersionInCodebase(old, new *version.Version) error { - filesToChange, err := getListOfFilesInPaths([]string{ - "./.github/workflows", +func replaceGoVersionInCodebase(old, new *version.Version, workflowUpdate bool) error { + explore := []string{ "./test/templates", "./build.env", "./docker/bootstrap/Dockerfile.common", - }) + } + if workflowUpdate { + explore = append(explore, "./.github/workflows") + } + filesToChange, err := getListOfFilesInPaths(explore) if err != nil { return err } From b5b6bea52f64aed3e88e317adaaf76b8fc1e610a Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 10:57:34 +0200 Subject: [PATCH 11/34] better PR message Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 41 +++++++++------------ 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 92ea4665beb..063b4c14eb6 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -24,9 +24,9 @@ jobs: id: detect-and-update run: | if [ ${{github.ref}} == "main" ]; then - go run ./go/tools/go-upgrade/go_upgrade.go --main --allow-major-upgrade + go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update --main --allow-major-upgrade else - go run ./go/tools/go-upgrade/go_upgrade.go + go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update fi output=$(git status -s) @@ -34,26 +34,21 @@ jobs: exit 0 fi echo "create-pr=true" >> $GITHUB_OUTPUT - git add --all - git commit -m + + go_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_go_version) + bootstrap_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_bootstrap_version) + echo "go-version=${go_version}" >> $GITHUB_OUTPUT + echo "bootstrap-version=${bootstrap_version}" >> $GITHUB_OUTPUT - - name: Create pull request - uses: devops-infra/action-pull-request@v0.5.5 + - name: Create Pull Request + if: steps.detect-and-update.outputs.create-pr == 'true' + uses: peter-evans/create-pull-request@v4 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - title: Automatic pull request - target_branch: main - reviewer: frouioui - -# - name: Create Pull Request -# if: steps.detect-and-update.outputs.create-pr == 'true' -# uses: peter-evans/create-pull-request@v4 -# with: -# commit-message: "Upgrade the Golang version" -# signoff: true -# delete-branch: true -# title: "Bump the Golang version" -# body: "test" -# base: "main" # TODO: remove, this is for test purposes -# reviewers: | -# frouioui + commit-message: "bump go version to go${steps.detect-and-update.outputs.go-version}" + signoff: true + delete-branch: true + title: "Upgrade the Golang version to `go${steps.detect-and-update.outputs.go-version}`" + body: "This Pull Request bumps the Golang version to `go${steps.detect-and-update.outputs.go-version}` and the bootstrap version to `${steps.detect-and-update.outputs.bootstrap-version}`." + base: "main" # TODO: remove, this is for test purposes + reviewers: | + frouioui From f2366ffb01d5027171292834266f5dde07ad6e37 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 14:19:38 +0200 Subject: [PATCH 12/34] fix no-workflow-update flag and better PR body msg Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 8 +++++++- go/tools/go-upgrade/go_upgrade.go | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 063b4c14eb6..e158ee61e8a 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -48,7 +48,13 @@ jobs: signoff: true delete-branch: true title: "Upgrade the Golang version to `go${steps.detect-and-update.outputs.go-version}`" - body: "This Pull Request bumps the Golang version to `go${steps.detect-and-update.outputs.go-version}` and the bootstrap version to `${steps.detect-and-update.outputs.bootstrap-version}`." + body: | + This Pull Request bumps the Golang version to `go${steps.detect-and-update.outputs.go-version}` and the bootstrap version to `${steps.detect-and-update.outputs.bootstrap-version}`. + + There are a few manual steps remaining: + - Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. + - Update the `./.github/workflows` file with the newer Golang version, the bot, also, cannot handle that due to permissions. + - To accomplish this, run the following: `` base: "main" # TODO: remove, this is for test purposes reviewers: | frouioui diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 2b15c09c457..362095302f8 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -201,13 +201,13 @@ func chooseNewVersion(curVersion *version.Version, latestVersions version.Collec // replaceGoVersionInCodebase goes through all the files in the codebase where the // Golang version must be updated -func replaceGoVersionInCodebase(old, new *version.Version, workflowUpdate bool) error { +func replaceGoVersionInCodebase(old, new *version.Version, noWorkflowUpdate bool) error { explore := []string{ "./test/templates", "./build.env", "./docker/bootstrap/Dockerfile.common", } - if workflowUpdate { + if !noWorkflowUpdate { explore = append(explore, "./.github/workflows") } filesToChange, err := getListOfFilesInPaths(explore) From cbebdc3dd7d6e8e5513159f0b8a58403bd425ed6 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 14:40:15 +0200 Subject: [PATCH 13/34] fix PR title, branch and body format Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index e158ee61e8a..56ae150b7be 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -1,7 +1,6 @@ name: Update Golang Version on: - # Triggers the workflow every day schedule: - cron: "0 0 * * *" workflow_dispatch: @@ -9,6 +8,9 @@ on: jobs: update_golang_version: +# strategy: +# matrix: +# branches: [ main, release-16.0, release-15.0, release-14.0 ] name: Update Golang Version runs-on: ubuntu-latest steps: @@ -44,12 +46,13 @@ jobs: if: steps.detect-and-update.outputs.create-pr == 'true' uses: peter-evans/create-pull-request@v4 with: - commit-message: "bump go version to go${steps.detect-and-update.outputs.go-version}" + branch: "upgrade-go-to-${{steps.detect-and-update.outputs.go-version}}-on-${{github.ref}}" + commit-message: "bump go version to go${{steps.detect-and-update.outputs.go-version}}" signoff: true delete-branch: true - title: "Upgrade the Golang version to `go${steps.detect-and-update.outputs.go-version}`" + title: "Upgrade the Golang version to `go${{steps.detect-and-update.outputs.go-version}}`" body: | - This Pull Request bumps the Golang version to `go${steps.detect-and-update.outputs.go-version}` and the bootstrap version to `${steps.detect-and-update.outputs.bootstrap-version}`. + This Pull Request bumps the Golang version to `go${{steps.detect-and-update.outputs.go-version}}` and the bootstrap version to `${{steps.detect-and-update.outputs.bootstrap-version}}`. There are a few manual steps remaining: - Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. From 1857ee381497dcdf35d344f03278b0e34a954a72 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 14:54:53 +0200 Subject: [PATCH 14/34] test matrix and pwd path Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 104 ++++++++++++-------- 1 file changed, 64 insertions(+), 40 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 56ae150b7be..9a39a5c5a9a 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -8,9 +8,10 @@ on: jobs: update_golang_version: -# strategy: -# matrix: -# branches: [ main, release-16.0, release-15.0, release-14.0 ] + strategy: + matrix: + branch: [ main ] +# branch: [ main, release-16.0, release-15.0, release-14.0 ] name: Update Golang Version runs-on: ubuntu-latest steps: @@ -19,45 +20,68 @@ jobs: with: go-version: 1.20.2 - - name: Check out code into the Go module directory + - name: Check out code uses: actions/checkout@v3 - - name: Detect new version and update codebase - id: detect-and-update + - name: Detect new version run: | - if [ ${{github.ref}} == "main" ]; then - go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update --main --allow-major-upgrade - else - go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update - fi - - output=$(git status -s) - if [ -z "${output}" ]; then - exit 0 - fi - echo "create-pr=true" >> $GITHUB_OUTPUT - - go_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_go_version) - bootstrap_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_bootstrap_version) - echo "go-version=${go_version}" >> $GITHUB_OUTPUT - echo "bootstrap-version=${bootstrap_version}" >> $GITHUB_OUTPUT + pwd + ls - - name: Create Pull Request - if: steps.detect-and-update.outputs.create-pr == 'true' - uses: peter-evans/create-pull-request@v4 + - name: Check out matrix branch + uses: actions/checkout@v3 with: - branch: "upgrade-go-to-${{steps.detect-and-update.outputs.go-version}}-on-${{github.ref}}" - commit-message: "bump go version to go${{steps.detect-and-update.outputs.go-version}}" - signoff: true - delete-branch: true - title: "Upgrade the Golang version to `go${{steps.detect-and-update.outputs.go-version}}`" - body: | - This Pull Request bumps the Golang version to `go${{steps.detect-and-update.outputs.go-version}}` and the bootstrap version to `${{steps.detect-and-update.outputs.bootstrap-version}}`. - - There are a few manual steps remaining: - - Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. - - Update the `./.github/workflows` file with the newer Golang version, the bot, also, cannot handle that due to permissions. - - To accomplish this, run the following: `` - base: "main" # TODO: remove, this is for test purposes - reviewers: | - frouioui + ref: ${{ matrix.branch }} + path: './vitess-to-upgrade/' + + - name: Detect new version and update codebase + id: detect-and-update + run: | + pwd + ls + cd .. + pwd + ls +# if [ ${{github.ref}} == "main" ]; then +# go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update --main --allow-major-upgrade +# else +# go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update +# fi +# +# output=$(git status -s) +# if [ -z "${output}" ]; then +# exit 0 +# fi +# echo "create-pr=true" >> $GITHUB_OUTPUT +# +# go_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_go_version) +# bootstrap_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_bootstrap_version) +# echo "go-version=${go_version}" >> $GITHUB_OUTPUT +# echo "bootstrap-version=${bootstrap_version}" >> $GITHUB_OUTPUT +# +# - name: Create Pull Request +# if: steps.detect-and-update.outputs.create-pr == 'true' +# uses: peter-evans/create-pull-request@v4 +# with: +# branch: "upgrade-go-to-${{steps.detect-and-update.outputs.go-version}}-on-${{github.ref}}" +# commit-message: "bump go version to go${{steps.detect-and-update.outputs.go-version}}" +# signoff: true +# delete-branch: true +# title: "Upgrade the Golang version to `go${{steps.detect-and-update.outputs.go-version}}`" +# body: | +# This Pull Request bumps the Golang version to `go${{steps.detect-and-update.outputs.go-version}}` and the bootstrap version to `${{steps.detect-and-update.outputs.bootstrap-version}}`. +# +# There are a few manual steps remaining: +# - Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. +# - Update the `./.github/workflows` file with the newer Golang version, the bot, also, cannot handle that due to permissions. +# - To accomplish this, run the following: `` +# base: "main" # TODO: remove, this is for test purposes +# reviewers: | +# frouioui +# labels: | +# Skip CI +# go +# Benchmark me +# Component: General +# Type: CI/Build +# From 6bcbee5db442e81f0f928a83275d0e7c56463ad0 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 16:40:58 +0200 Subject: [PATCH 15/34] enhanced matrix Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 98 ++++++++++----------- go.mod | 2 +- go.sum | 1 - 3 files changed, 46 insertions(+), 55 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 9a39a5c5a9a..2ab0fb499d9 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -23,11 +23,6 @@ jobs: - name: Check out code uses: actions/checkout@v3 - - name: Detect new version - run: | - pwd - ls - - name: Check out matrix branch uses: actions/checkout@v3 with: @@ -37,51 +32,48 @@ jobs: - name: Detect new version and update codebase id: detect-and-update run: | - pwd - ls - cd .. - pwd - ls -# if [ ${{github.ref}} == "main" ]; then -# go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update --main --allow-major-upgrade -# else -# go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update -# fi -# -# output=$(git status -s) -# if [ -z "${output}" ]; then -# exit 0 -# fi -# echo "create-pr=true" >> $GITHUB_OUTPUT -# -# go_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_go_version) -# bootstrap_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_bootstrap_version) -# echo "go-version=${go_version}" >> $GITHUB_OUTPUT -# echo "bootstrap-version=${bootstrap_version}" >> $GITHUB_OUTPUT -# -# - name: Create Pull Request -# if: steps.detect-and-update.outputs.create-pr == 'true' -# uses: peter-evans/create-pull-request@v4 -# with: -# branch: "upgrade-go-to-${{steps.detect-and-update.outputs.go-version}}-on-${{github.ref}}" -# commit-message: "bump go version to go${{steps.detect-and-update.outputs.go-version}}" -# signoff: true -# delete-branch: true -# title: "Upgrade the Golang version to `go${{steps.detect-and-update.outputs.go-version}}`" -# body: | -# This Pull Request bumps the Golang version to `go${{steps.detect-and-update.outputs.go-version}}` and the bootstrap version to `${{steps.detect-and-update.outputs.bootstrap-version}}`. -# -# There are a few manual steps remaining: -# - Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. -# - Update the `./.github/workflows` file with the newer Golang version, the bot, also, cannot handle that due to permissions. -# - To accomplish this, run the following: `` -# base: "main" # TODO: remove, this is for test purposes -# reviewers: | -# frouioui -# labels: | -# Skip CI -# go -# Benchmark me -# Component: General -# Type: CI/Build -# + cd ./vitess-to-upgrade + if [ ${{github.ref}} == "main" ]; then + go run ../go/tools/go-upgrade/go_upgrade.go --no-workflow-update --main --allow-major-upgrade + else + go run ../go/tools/go-upgrade/go_upgrade.go --no-workflow-update + fi + + output=$(git status -s) + if [ -z "${output}" ]; then + exit 0 + fi + echo "create-pr=true" >> $GITHUB_OUTPUT + + go_version=$(go run ../go/tools/go-upgrade/go_upgrade.go get_go_version) + bootstrap_version=$(go run ../go/tools/go-upgrade/go_upgrade.go get_bootstrap_version) + echo "go-version=${go_version}" >> $GITHUB_OUTPUT + echo "bootstrap-version=${bootstrap_version}" >> $GITHUB_OUTPUT + + - name: Create Pull Request + if: steps.detect-and-update.outputs.create-pr == 'true' + uses: peter-evans/create-pull-request@v4 + with: + path: ./vitess-to-upgrade + branch: "upgrade-go-to-${{steps.detect-and-update.outputs.go-version}}-on-${{ matrix.branch }}" + commit-message: "bump go version to go${{steps.detect-and-update.outputs.go-version}}" + signoff: true + delete-branch: true + title: "Upgrade the Golang version to `go${{steps.detect-and-update.outputs.go-version}}`" + body: | + This Pull Request bumps the Golang version to `go${{steps.detect-and-update.outputs.go-version}}` and the bootstrap version to `${{steps.detect-and-update.outputs.bootstrap-version}}`. + + There are a few manual steps remaining: + - Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. + - Update the `./.github/workflows` file with the newer Golang version, the bot, also, cannot handle that due to permissions. + - To accomplish this, run the following: `` + base: ${{ matrix.branch }} + reviewers: | + frouioui + labels: | + Skip CI + go + Benchmark me + Component: General + Type: CI/Build + diff --git a/go.mod b/go.mod index aa8f87bef0c..f8ebc88192a 100644 --- a/go.mod +++ b/go.mod @@ -109,6 +109,7 @@ require ( require ( github.com/bndr/gotabulate v1.1.2 + github.com/hashicorp/go-version v1.6.0 github.com/kr/pretty v0.3.1 github.com/kr/text v0.2.0 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 @@ -154,7 +155,6 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v1.4.0 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 43b1be37384..077014f19d3 100644 --- a/go.sum +++ b/go.sum @@ -458,7 +458,6 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= From 4c045176521b6f9dfcb04ef1f71251b4090e7376 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 17:15:08 +0200 Subject: [PATCH 16/34] update_workflows option Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 8 +++-- go/tools/go-upgrade/go_upgrade.go | 39 ++++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 2ab0fb499d9..d1396f06c13 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -32,6 +32,9 @@ jobs: - name: Detect new version and update codebase id: detect-and-update run: | + old_go_version=$(go run ../go/tools/go-upgrade/go_upgrade.go get_go_version) + echo "old-go-version=${old_go_version}" >> $GITHUB_OUTPUT + cd ./vitess-to-upgrade if [ ${{github.ref}} == "main" ]; then go run ../go/tools/go-upgrade/go_upgrade.go --no-workflow-update --main --allow-major-upgrade @@ -64,8 +67,9 @@ jobs: This Pull Request bumps the Golang version to `go${{steps.detect-and-update.outputs.go-version}}` and the bootstrap version to `${{steps.detect-and-update.outputs.bootstrap-version}}`. There are a few manual steps remaining: - - Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. - - Update the `./.github/workflows` file with the newer Golang version, the bot, also, cannot handle that due to permissions. + - [ ] Make sure you update the Golang version used in the previous and next release branches for the Upgrade/Downgrade tests. + - [ ] Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. + - [ ] Update the `./.github/workflows/*.yml` files with the newer Golang version, the bot cannot handle that due to permissions. - To accomplish this, run the following: `` base: ${{ matrix.branch }} reviewers: | diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 362095302f8..633c68d04fc 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -45,6 +45,8 @@ func main() { noWorkflowUpdate := flag.Bool("no-workflow-update", false, "Whether or not the workflow files should be updated. Useful when using this script to auto-create PRs.") allowMajorUpgrade := flag.Bool("allow-major-upgrade", false, "Defines if Golang major version upgrade are allowed.") isMainBranch := flag.Bool("main", false, "Defines if the current branch is the main branch.") + goFrom := flag.String("go-from", "", "The original Golang version we start with.") + goTo := flag.String("go-to", "", "The Golang version we want to upgrade to.") flag.Parse() switch strings.ToLower(os.Args[1]) { @@ -60,6 +62,14 @@ func main() { log.Fatal(err) } fmt.Println(currentBootstrapVersionF) + case "update_workflows": + if !(*noWorkflowUpdate) { + break + } + err := updateWorkflowFilesOnly(*goFrom, *goTo) + if err != nil { + log.Fatal(err) + } default: err := upgradePath(*allowMajorUpgrade, *noWorkflowUpdate, *isMainBranch) if err != nil { @@ -68,6 +78,33 @@ func main() { } } +func updateWorkflowFilesOnly(goFrom, goTo string) error { + oldV, err := version.NewVersion(goFrom) + if err != nil { + return err + } + newV, err := version.NewVersion(goTo) + if err != nil { + return err + } + filesToChange, err := getListOfFilesInPaths([]string{"./.github/workflows"}) + if err != nil { + return err + } + + for _, fileToChange := range filesToChange { + err = replaceInFile( + []string{"go-version: " + oldV.String()}, + []string{"go-version: " + newV.String()}, + fileToChange, + ) + if err != nil { + return err + } + } + return nil +} + func upgradePath(allowMajorUpgrade, noWorkflowUpdate, isMainBranch bool) error { currentVersion, err := currentGolangVersion() if err != nil { @@ -111,7 +148,7 @@ func upgradePath(allowMajorUpgrade, noWorkflowUpdate, isMainBranch bool) error { // // The file `./build.env` describes which version of Golang is expected by Vitess. // We use this file to detect the current Golang version of our codebase. -// The file contains `goversion_min 1.20.1`, we will grep `goversion_min` to finally find +// The file contains `goversion_min x.xx.xx`, we will grep `goversion_min` to finally find // the precise golang version we're using. func currentGolangVersion() (*version.Version, error) { contentRaw, err := os.ReadFile("build.env") From 3687eddd97adb33708473a4a8c094625ece0b648 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 17:30:17 +0200 Subject: [PATCH 17/34] update workflows independently Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 2 +- go/tools/go-upgrade/go_upgrade.go | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index d1396f06c13..92d2755e864 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -70,7 +70,7 @@ jobs: - [ ] Make sure you update the Golang version used in the previous and next release branches for the Upgrade/Downgrade tests. - [ ] Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. - [ ] Update the `./.github/workflows/*.yml` files with the newer Golang version, the bot cannot handle that due to permissions. - - To accomplish this, run the following: `` + - To accomplish this, run the following: `go run ./go/tools/go-upgrade/go_upgrade.go --go-from=${{steps.detect-and-update.outputs.old-go-version}} --go-to=${{steps.detect-and-update.outputs.go-version}} update_workflows` base: ${{ matrix.branch }} reviewers: | frouioui diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 633c68d04fc..272fe9bb868 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -30,6 +30,7 @@ import ( "time" "github.com/hashicorp/go-version" + "golang.org/x/exp/slices" ) const ( @@ -49,21 +50,21 @@ func main() { goTo := flag.String("go-to", "", "The Golang version we want to upgrade to.") flag.Parse() - switch strings.ToLower(os.Args[1]) { - case "get_go_version": + switch { + case slices.Contains(os.Args, "get_go_version"): currentVersion, err := currentGolangVersion() if err != nil { log.Fatal(err) } fmt.Println(currentVersion.String()) - case "get_bootstrap_version": + case slices.Contains(os.Args, "get_bootstrap_version"): currentBootstrapVersionF, err := currentBootstrapVersion() if err != nil { log.Fatal(err) } fmt.Println(currentBootstrapVersionF) - case "update_workflows": - if !(*noWorkflowUpdate) { + case slices.Contains(os.Args, "update_workflows"): + if *noWorkflowUpdate { break } err := updateWorkflowFilesOnly(*goFrom, *goTo) From ec20bb562c1ce10bd0e2125fbc891ee04fe4dffa Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 17:56:13 +0200 Subject: [PATCH 18/34] test without matrix Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 27 +++++++-------------- 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 92d2755e864..c6b443d9b34 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -8,9 +8,8 @@ on: jobs: update_golang_version: - strategy: - matrix: - branch: [ main ] +# strategy: +# matrix: # branch: [ main, release-16.0, release-15.0, release-14.0 ] name: Update Golang Version runs-on: ubuntu-latest @@ -23,23 +22,16 @@ jobs: - name: Check out code uses: actions/checkout@v3 - - name: Check out matrix branch - uses: actions/checkout@v3 - with: - ref: ${{ matrix.branch }} - path: './vitess-to-upgrade/' - - name: Detect new version and update codebase id: detect-and-update run: | - old_go_version=$(go run ../go/tools/go-upgrade/go_upgrade.go get_go_version) + old_go_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_go_version) echo "old-go-version=${old_go_version}" >> $GITHUB_OUTPUT - cd ./vitess-to-upgrade if [ ${{github.ref}} == "main" ]; then - go run ../go/tools/go-upgrade/go_upgrade.go --no-workflow-update --main --allow-major-upgrade + go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update --main --allow-major-upgrade else - go run ../go/tools/go-upgrade/go_upgrade.go --no-workflow-update + go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update fi output=$(git status -s) @@ -48,8 +40,8 @@ jobs: fi echo "create-pr=true" >> $GITHUB_OUTPUT - go_version=$(go run ../go/tools/go-upgrade/go_upgrade.go get_go_version) - bootstrap_version=$(go run ../go/tools/go-upgrade/go_upgrade.go get_bootstrap_version) + go_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_go_version) + bootstrap_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_bootstrap_version) echo "go-version=${go_version}" >> $GITHUB_OUTPUT echo "bootstrap-version=${bootstrap_version}" >> $GITHUB_OUTPUT @@ -57,8 +49,7 @@ jobs: if: steps.detect-and-update.outputs.create-pr == 'true' uses: peter-evans/create-pull-request@v4 with: - path: ./vitess-to-upgrade - branch: "upgrade-go-to-${{steps.detect-and-update.outputs.go-version}}-on-${{ matrix.branch }}" + branch: "upgrade-go-to-${{steps.detect-and-update.outputs.go-version}}-on-${{github.ref}}" #${{ matrix.branch }} commit-message: "bump go version to go${{steps.detect-and-update.outputs.go-version}}" signoff: true delete-branch: true @@ -71,7 +62,7 @@ jobs: - [ ] Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. - [ ] Update the `./.github/workflows/*.yml` files with the newer Golang version, the bot cannot handle that due to permissions. - To accomplish this, run the following: `go run ./go/tools/go-upgrade/go_upgrade.go --go-from=${{steps.detect-and-update.outputs.old-go-version}} --go-to=${{steps.detect-and-update.outputs.go-version}} update_workflows` - base: ${{ matrix.branch }} + base: main #${{ matrix.branch }} reviewers: | frouioui labels: | From 32315a8981e20a0d01c03446fbf1ad229420dc05 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 9 Mar 2023 19:35:10 +0200 Subject: [PATCH 19/34] Add docs to go_upgrade and re-add matrix to the workflow Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 20 ++++---- go/tools/go-upgrade/go_upgrade.go | 53 +++++++++++++++++++-- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index c6b443d9b34..e31d6ccd1b1 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -4,13 +4,12 @@ on: schedule: - cron: "0 0 * * *" workflow_dispatch: - pull_request: jobs: update_golang_version: -# strategy: -# matrix: -# branch: [ main, release-16.0, release-15.0, release-14.0 ] + strategy: + matrix: + branch: [ main, release-16.0, release-15.0, release-14.0 ] name: Update Golang Version runs-on: ubuntu-latest steps: @@ -21,6 +20,8 @@ jobs: - name: Check out code uses: actions/checkout@v3 + with: + ref: ${{ matrix.branch }} - name: Detect new version and update codebase id: detect-and-update @@ -28,7 +29,7 @@ jobs: old_go_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_go_version) echo "old-go-version=${old_go_version}" >> $GITHUB_OUTPUT - if [ ${{github.ref}} == "main" ]; then + if [ ${{ matrix.branch }} == "main" ]; then go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update --main --allow-major-upgrade else go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update @@ -49,11 +50,11 @@ jobs: if: steps.detect-and-update.outputs.create-pr == 'true' uses: peter-evans/create-pull-request@v4 with: - branch: "upgrade-go-to-${{steps.detect-and-update.outputs.go-version}}-on-${{github.ref}}" #${{ matrix.branch }} + branch: "upgrade-go-to-${{steps.detect-and-update.outputs.go-version}}-on-${{ matrix.branch }}" commit-message: "bump go version to go${{steps.detect-and-update.outputs.go-version}}" signoff: true delete-branch: true - title: "Upgrade the Golang version to `go${{steps.detect-and-update.outputs.go-version}}`" + title: "[${{ matrix.branch }}] Upgrade the Golang version to `go${{steps.detect-and-update.outputs.go-version}}`" body: | This Pull Request bumps the Golang version to `go${{steps.detect-and-update.outputs.go-version}}` and the bootstrap version to `${{steps.detect-and-update.outputs.bootstrap-version}}`. @@ -62,13 +63,10 @@ jobs: - [ ] Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. - [ ] Update the `./.github/workflows/*.yml` files with the newer Golang version, the bot cannot handle that due to permissions. - To accomplish this, run the following: `go run ./go/tools/go-upgrade/go_upgrade.go --go-from=${{steps.detect-and-update.outputs.old-go-version}} --go-to=${{steps.detect-and-update.outputs.go-version}} update_workflows` - base: main #${{ matrix.branch }} - reviewers: | - frouioui + base: ${{ matrix.branch }} labels: | Skip CI go Benchmark me Component: General Type: CI/Build - diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 272fe9bb868..9df31dd4daf 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -17,11 +17,8 @@ limitations under the License. package main import ( - "encoding/json" - "flag" "fmt" "io" - "log" "net/http" "os" "path" @@ -29,10 +26,60 @@ import ( "strings" "time" + "encoding/json" + "flag" + "log" + "github.com/hashicorp/go-version" "golang.org/x/exp/slices" ) +// The tool go_upgrade allows us to automate some tasks required +// to bump the version of Golang used throughout our codebase. +// +// It comes with four commands: +// - Default: `./go/tools/go-upgrade/go_upgrade.go` +// +// This will simply bump the version across the codebase. +// The latest available version of Golang will be fetched +// and used instead of the old version. +// +// By default, we do allow major Golang version upgrade such +// as 1.20 to 1.21 but this can be overridden using the +// allow-major-upgrade CLI flag. Usually, we only allow such +// upgrade on the main branch of the repository. +// +// In CI, particularly, we do not want to modify the workflow +// files before automatically creating a Pull Request to +// avoid permission issues. The rewrite of workflow files can +// be disabled using the no-workflow-update CLI flag. +// +// Moreover, this command automatically bumps the bootstrap +// version of our codebase. If we are on the main branch, we +// want to use the CLI flag `main` to remember to increment +// the bootstrap version by 1 instead of 0.1. +// +// +// - Get Current Go Version: `./go/tools/go-upgrade/go_upgrade.go get_go_version` +// +// This command outputs the currently used Golang version of +// our codebase. +// +// +// - Get Current bootstrap Version: `./go/tools/go-upgrade/go_upgrade.go get_bootstrap_version` +// +// This command outputs the currently used bootstrap version +// of our codebase. +// +// +// - Update Workflows: `./go/tools/go-upgrade/go_upgrade.go update_workflows` +// +// This command is used after the auto Pull Request has been +// created. One of the reviewer must go and use this command +// to update the workflows files with the newer Golang version. +// It takes two arguments `go-from` and `go-to` to identify the +// old Golang version (go-from) and the new one (go-to). + const ( goDevAPI = "https://go.dev/dl/?mode=json" ) From 8d3ed022547c3d51040a643c18b2752d77be7de1 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 14 Mar 2023 08:37:47 +0200 Subject: [PATCH 20/34] support major version upgrade Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 9df31dd4daf..4736a23e3f4 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -270,7 +270,7 @@ func getLatestStableGolangReleases() (version.Collection, error) { func chooseNewVersion(curVersion *version.Version, latestVersions version.Collection, allowMajorUpgrade bool) *version.Version { selectedVersion := curVersion for _, latestVersion := range latestVersions { - if !allowMajorUpgrade && !isSameMajorVersion(latestVersion, selectedVersion) { + if !allowMajorUpgrade && !isSameMajorMinorVersion(latestVersion, selectedVersion) { continue } if latestVersion.GreaterThan(selectedVersion) { @@ -311,7 +311,7 @@ func replaceGoVersionInCodebase(old, new *version.Version, noWorkflowUpdate bool } } - if !isSameMajorVersion(old, new) { + if !isSameMajorMinorVersion(old, new) { err = replaceInFile( []string{fmt.Sprintf("%d.%d", old.Segments()[0], old.Segments()[1])}, []string{fmt.Sprintf("%d.%d", new.Segments()[0], new.Segments()[1])}, @@ -394,8 +394,8 @@ func updateBootstrapChangelog(new float64, goVersion *version.Version) error { return nil } -func isSameMajorVersion(a, b *version.Version) bool { - return a.Segments()[1] == b.Segments()[1] +func isSameMajorMinorVersion(a, b *version.Version) bool { + return a.Segments()[0] == b.Segments()[0] && a.Segments()[1] == b.Segments()[1] } func getListOfFilesInPaths(pathsToExplore []string) ([]string, error) { From a1f71bfd8a37de8520e8feb936a27d643b4ac109 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 14 Mar 2023 08:43:29 +0200 Subject: [PATCH 21/34] do not update bootstrap version if they are the same Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 4736a23e3f4..1512254853a 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -325,6 +325,9 @@ func replaceGoVersionInCodebase(old, new *version.Version, noWorkflowUpdate bool } func updateBootstrapVersionInCodebase(old, new float64, newGoVersion *version.Version) error { + if old == new { + return nil + } files, err := getListOfFilesInPaths([]string{ "./docker/base", "./docker/lite", From 0da984672b34dcc4808372cf00255272ca9f0c80 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 14 Mar 2023 14:40:58 +0200 Subject: [PATCH 22/34] migrate to cobra instead of pflags Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 159 +++++++++++++++++++++++------- 1 file changed, 123 insertions(+), 36 deletions(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 1512254853a..724b393a419 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -19,6 +19,7 @@ package main import ( "fmt" "io" + "log" "net/http" "os" "path" @@ -27,11 +28,9 @@ import ( "time" "encoding/json" - "flag" - "log" "github.com/hashicorp/go-version" - "golang.org/x/exp/slices" + "github.com/spf13/cobra" ) // The tool go_upgrade allows us to automate some tasks required @@ -89,40 +88,128 @@ type latestGolangRelease struct { Stable bool `json:"stable"` } +var ( + noWorkflowUpdate = false + allowMajorUpgrade = false + isMainBranch = false + goFrom = "" + goTo = "" + + rootCmd = &cobra.Command{ + Use: "go-upgrade", + Short: "Automates the Golang upgrade.", + Long: `go-upgrade allows us to automate some tasks required to bump the version of Golang used throughout our codebase. + +It mostly used by the update_golang_version.yml CI workflow that runs on a CRON. +`, + Run: func(cmd *cobra.Command, args []string) { + _ = cmd.Help() + }, + Args: cobra.NoArgs, + } + + getCmd = &cobra.Command{ + Use: "get", + Short: "Command to get useful information about the codebase.", + Long: "Command to get useful information about the codebase.", + Run: func(cmd *cobra.Command, args []string) { + _ = cmd.Help() + }, + Args: cobra.NoArgs, + } + + getGoCmd = &cobra.Command{ + Use: "go-version", + Short: "go-version prints the Golang version used by the current codebase.", + Long: "go-version prints the Golang version used by the current codebase.", + Run: runGetGoCmd, + Args: cobra.NoArgs, + } + + getBootstrapCmd = &cobra.Command{ + Use: "bootstrap-version", + Short: "bootstrap-version prints the Docker Bootstrap version used by the current codebase.", + Long: "bootstrap-version prints the Docker Bootstrap version used by the current codebase.", + Run: runGetBootstrapCmd, + Args: cobra.NoArgs, + } + + upgradeCmd = &cobra.Command{ + Use: "upgrade", + Short: "upgrade will upgrade the Golang and Bootstrap versions of the codebase to the latest available version.", + Long: `This command bumps the Golang and Bootstrap versions of the codebase. + +The latest available version of Golang will be fetched and used instead of the old version. + +By default, we do not allow major Golang version upgrade such as 1.20 to 1.21 but this can be overridden using the +--allow-major-upgrade CLI flag. Usually, we only allow such upgrade on the main branch of the repository. + +In CI, particularly, we do not want to modify the workflow files before automatically creating a Pull Request to +avoid permission issues. The rewrite of workflow files can be disabled using the --no-workflow-update CLI flag. + +Moreover, this command automatically bumps the bootstrap version of our codebase. If we are on the main branch, we +want to use the CLI flag --main to remember to increment the bootstrap version by 1 instead of 0.1.`, + Run: runUpgradeCmd, + Args: cobra.NoArgs, + } + + upgradeWorkflowsCmd = &cobra.Command{ + Use: "workflows", + Short: "workflows will upgrade the Golang version used in our CI workflows files.", + Long: "This step is omitted by the bot since. We let the maintainers of Vitess manually upgrade the version used by the workflows using this command.", + Run: runUpgradeWorkflowsCmd, + Args: cobra.NoArgs, + } +) + +func init() { + rootCmd.AddCommand(getCmd) + rootCmd.AddCommand(upgradeCmd) + + getCmd.AddCommand(getGoCmd) + getCmd.AddCommand(getBootstrapCmd) + + upgradeCmd.AddCommand(upgradeWorkflowsCmd) + + upgradeCmd.Flags().BoolVar(&noWorkflowUpdate, "no-workflow-update", noWorkflowUpdate, "Whether or not the workflow files should be updated. Useful when using this script to auto-create PRs.") + upgradeCmd.Flags().BoolVar(&allowMajorUpgrade, "allow-major-upgrade", allowMajorUpgrade, "Defines if Golang major version upgrade are allowed.") + upgradeCmd.Flags().BoolVar(&isMainBranch, "main", isMainBranch, "Defines if the current branch is the main branch.") + + upgradeWorkflowsCmd.Flags().StringVar(&goFrom, "go-from", goFrom, "The original Golang version we start with.") + upgradeWorkflowsCmd.Flags().StringVar(&goTo, "go-to", goTo, "The Golang version we want to upgrade to.") +} + func main() { - noWorkflowUpdate := flag.Bool("no-workflow-update", false, "Whether or not the workflow files should be updated. Useful when using this script to auto-create PRs.") - allowMajorUpgrade := flag.Bool("allow-major-upgrade", false, "Defines if Golang major version upgrade are allowed.") - isMainBranch := flag.Bool("main", false, "Defines if the current branch is the main branch.") - goFrom := flag.String("go-from", "", "The original Golang version we start with.") - goTo := flag.String("go-to", "", "The Golang version we want to upgrade to.") - flag.Parse() - - switch { - case slices.Contains(os.Args, "get_go_version"): - currentVersion, err := currentGolangVersion() - if err != nil { - log.Fatal(err) - } - fmt.Println(currentVersion.String()) - case slices.Contains(os.Args, "get_bootstrap_version"): - currentBootstrapVersionF, err := currentBootstrapVersion() - if err != nil { - log.Fatal(err) - } - fmt.Println(currentBootstrapVersionF) - case slices.Contains(os.Args, "update_workflows"): - if *noWorkflowUpdate { - break - } - err := updateWorkflowFilesOnly(*goFrom, *goTo) - if err != nil { - log.Fatal(err) - } - default: - err := upgradePath(*allowMajorUpgrade, *noWorkflowUpdate, *isMainBranch) - if err != nil { - log.Fatal(err) - } + cobra.CheckErr(rootCmd.Execute()) +} + +func runGetGoCmd(_ *cobra.Command, _ []string) { + currentVersion, err := currentGolangVersion() + if err != nil { + log.Fatal(err) + } + fmt.Println(currentVersion.String()) +} + +func runGetBootstrapCmd(_ *cobra.Command, _ []string) { + currentVersion, err := currentBootstrapVersion() + if err != nil { + log.Fatal(err) + } + fmt.Println(currentVersion) +} + +func runUpgradeWorkflowsCmd(_ *cobra.Command, _ []string) { + err := updateWorkflowFilesOnly(goFrom, goTo) + if err != nil { + log.Fatal(err) + } +} + +func runUpgradeCmd(_ *cobra.Command, _ []string) { + err := upgradePath(allowMajorUpgrade, noWorkflowUpdate, isMainBranch) + if err != nil { + log.Fatal(err) } } From f7287cb0757a5f0bc5de572538ce3380e145c10b Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 14 Mar 2023 14:43:14 +0200 Subject: [PATCH 23/34] remove redundant comment Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 46 ------------------------------- 1 file changed, 46 deletions(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 724b393a419..5b11f48f311 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -33,52 +33,6 @@ import ( "github.com/spf13/cobra" ) -// The tool go_upgrade allows us to automate some tasks required -// to bump the version of Golang used throughout our codebase. -// -// It comes with four commands: -// - Default: `./go/tools/go-upgrade/go_upgrade.go` -// -// This will simply bump the version across the codebase. -// The latest available version of Golang will be fetched -// and used instead of the old version. -// -// By default, we do allow major Golang version upgrade such -// as 1.20 to 1.21 but this can be overridden using the -// allow-major-upgrade CLI flag. Usually, we only allow such -// upgrade on the main branch of the repository. -// -// In CI, particularly, we do not want to modify the workflow -// files before automatically creating a Pull Request to -// avoid permission issues. The rewrite of workflow files can -// be disabled using the no-workflow-update CLI flag. -// -// Moreover, this command automatically bumps the bootstrap -// version of our codebase. If we are on the main branch, we -// want to use the CLI flag `main` to remember to increment -// the bootstrap version by 1 instead of 0.1. -// -// -// - Get Current Go Version: `./go/tools/go-upgrade/go_upgrade.go get_go_version` -// -// This command outputs the currently used Golang version of -// our codebase. -// -// -// - Get Current bootstrap Version: `./go/tools/go-upgrade/go_upgrade.go get_bootstrap_version` -// -// This command outputs the currently used bootstrap version -// of our codebase. -// -// -// - Update Workflows: `./go/tools/go-upgrade/go_upgrade.go update_workflows` -// -// This command is used after the auto Pull Request has been -// created. One of the reviewer must go and use this command -// to update the workflows files with the newer Golang version. -// It takes two arguments `go-from` and `go-to` to identify the -// old Golang version (go-from) and the new one (go-to). - const ( goDevAPI = "https://go.dev/dl/?mode=json" ) From 223e72d72a67de9105621d3697ac56262be7f942 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 14 Mar 2023 14:46:35 +0200 Subject: [PATCH 24/34] safety net around replaceGoVersionInCodebase to avoid unrequired upgrade Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 5b11f48f311..f4ff06ddb96 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -328,6 +328,9 @@ func chooseNewVersion(curVersion *version.Version, latestVersions version.Collec // replaceGoVersionInCodebase goes through all the files in the codebase where the // Golang version must be updated func replaceGoVersionInCodebase(old, new *version.Version, noWorkflowUpdate bool) error { + if old.Equal(new) { + return nil + } explore := []string{ "./test/templates", "./build.env", From 960361916f4539cae9c4021cb246e928a4ef932f Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 14 Mar 2023 14:49:03 +0200 Subject: [PATCH 25/34] Remove double negation in workflowUpdate flag Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index f4ff06ddb96..4db694183bd 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -43,7 +43,7 @@ type latestGolangRelease struct { } var ( - noWorkflowUpdate = false + workflowUpdate = true allowMajorUpgrade = false isMainBranch = false goFrom = "" @@ -99,7 +99,7 @@ By default, we do not allow major Golang version upgrade such as 1.20 to 1.21 bu --allow-major-upgrade CLI flag. Usually, we only allow such upgrade on the main branch of the repository. In CI, particularly, we do not want to modify the workflow files before automatically creating a Pull Request to -avoid permission issues. The rewrite of workflow files can be disabled using the --no-workflow-update CLI flag. +avoid permission issues. The rewrite of workflow files can be disabled using the --workflow-update=false CLI flag. Moreover, this command automatically bumps the bootstrap version of our codebase. If we are on the main branch, we want to use the CLI flag --main to remember to increment the bootstrap version by 1 instead of 0.1.`, @@ -125,7 +125,7 @@ func init() { upgradeCmd.AddCommand(upgradeWorkflowsCmd) - upgradeCmd.Flags().BoolVar(&noWorkflowUpdate, "no-workflow-update", noWorkflowUpdate, "Whether or not the workflow files should be updated. Useful when using this script to auto-create PRs.") + upgradeCmd.Flags().BoolVar(&workflowUpdate, "workflow-update", workflowUpdate, "Whether or not the workflow files should be updated. Useful when using this script to auto-create PRs.") upgradeCmd.Flags().BoolVar(&allowMajorUpgrade, "allow-major-upgrade", allowMajorUpgrade, "Defines if Golang major version upgrade are allowed.") upgradeCmd.Flags().BoolVar(&isMainBranch, "main", isMainBranch, "Defines if the current branch is the main branch.") @@ -161,7 +161,7 @@ func runUpgradeWorkflowsCmd(_ *cobra.Command, _ []string) { } func runUpgradeCmd(_ *cobra.Command, _ []string) { - err := upgradePath(allowMajorUpgrade, noWorkflowUpdate, isMainBranch) + err := upgradePath(allowMajorUpgrade, workflowUpdate, isMainBranch) if err != nil { log.Fatal(err) } @@ -194,7 +194,7 @@ func updateWorkflowFilesOnly(goFrom, goTo string) error { return nil } -func upgradePath(allowMajorUpgrade, noWorkflowUpdate, isMainBranch bool) error { +func upgradePath(allowMajorUpgrade, workflowUpdate, isMainBranch bool) error { currentVersion, err := currentGolangVersion() if err != nil { return err @@ -210,7 +210,7 @@ func upgradePath(allowMajorUpgrade, noWorkflowUpdate, isMainBranch bool) error { return nil } - err = replaceGoVersionInCodebase(currentVersion, upgradeTo, noWorkflowUpdate) + err = replaceGoVersionInCodebase(currentVersion, upgradeTo, workflowUpdate) if err != nil { return err } @@ -327,7 +327,7 @@ func chooseNewVersion(curVersion *version.Version, latestVersions version.Collec // replaceGoVersionInCodebase goes through all the files in the codebase where the // Golang version must be updated -func replaceGoVersionInCodebase(old, new *version.Version, noWorkflowUpdate bool) error { +func replaceGoVersionInCodebase(old, new *version.Version, workflowUpdate bool) error { if old.Equal(new) { return nil } @@ -336,7 +336,7 @@ func replaceGoVersionInCodebase(old, new *version.Version, noWorkflowUpdate bool "./build.env", "./docker/bootstrap/Dockerfile.common", } - if !noWorkflowUpdate { + if workflowUpdate { explore = append(explore, "./.github/workflows") } filesToChange, err := getListOfFilesInPaths(explore) From 49b4d3252d1a18a2e24e4cd7fcb2ab3fde7b529e Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 14 Mar 2023 17:12:39 +0200 Subject: [PATCH 26/34] Use regexp instead of mere string comparison Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 54 +++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 4db694183bd..dd4cb9dd00e 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -23,6 +23,7 @@ import ( "net/http" "os" "path" + "regexp" "strconv" "strings" "time" @@ -129,7 +130,6 @@ func init() { upgradeCmd.Flags().BoolVar(&allowMajorUpgrade, "allow-major-upgrade", allowMajorUpgrade, "Defines if Golang major version upgrade are allowed.") upgradeCmd.Flags().BoolVar(&isMainBranch, "main", isMainBranch, "Defines if the current branch is the main branch.") - upgradeWorkflowsCmd.Flags().StringVar(&goFrom, "go-from", goFrom, "The original Golang version we start with.") upgradeWorkflowsCmd.Flags().StringVar(&goTo, "go-to", goTo, "The Golang version we want to upgrade to.") } @@ -154,7 +154,7 @@ func runGetBootstrapCmd(_ *cobra.Command, _ []string) { } func runUpgradeWorkflowsCmd(_ *cobra.Command, _ []string) { - err := updateWorkflowFilesOnly(goFrom, goTo) + err := updateWorkflowFilesOnly(goTo) if err != nil { log.Fatal(err) } @@ -167,11 +167,7 @@ func runUpgradeCmd(_ *cobra.Command, _ []string) { } } -func updateWorkflowFilesOnly(goFrom, goTo string) error { - oldV, err := version.NewVersion(goFrom) - if err != nil { - return err - } +func updateWorkflowFilesOnly(goTo string) error { newV, err := version.NewVersion(goTo) if err != nil { return err @@ -183,7 +179,7 @@ func updateWorkflowFilesOnly(goFrom, goTo string) error { for _, fileToChange := range filesToChange { err = replaceInFile( - []string{"go-version: " + oldV.String()}, + []*regexp.Regexp{regexp.MustCompile(`(?i).*go-version:[[:space:]]*([0-9.]+).*`)}, []string{"go-version: " + newV.String()}, fileToChange, ) @@ -245,10 +241,13 @@ func currentGolangVersion() (*version.Version, error) { return nil, err } content := string(contentRaw) - idxBegin := strings.Index(content, "goversion_min") + len("goversion_min") + 1 - idxFinish := strings.Index(content[idxBegin:], "||") + idxBegin - versionStr := strings.TrimSpace(content[idxBegin:idxFinish]) - return version.NewVersion(versionStr) + + versre := regexp.MustCompile("(?i).*goversion_min[[:space:]]*([0-9.]+).*") + versionStr := versre.FindStringSubmatch(content) + if len(versionStr) != 2 { + return nil, fmt.Errorf("malformatted error, got: %v", versionStr) + } + return version.NewVersion(versionStr[1]) } func currentBootstrapVersion() (float64, error) { @@ -257,10 +256,13 @@ func currentBootstrapVersion() (float64, error) { return 0, err } content := string(contentRaw) - idxBegin := strings.Index(content, "BOOTSTRAP_VERSION=") + len("BOOTSTRAP_VERSION=") - idxFinish := strings.IndexByte(content[idxBegin:], '\n') + idxBegin - versionStr := strings.TrimSpace(content[idxBegin:idxFinish]) - f, err := strconv.ParseFloat(versionStr, 64) + + versre := regexp.MustCompile("(?i).*BOOTSTRAP_VERSION[[:space:]]*=[[:space:]]*([0-9.]+).*") + versionStr := versre.FindStringSubmatch(content) + if len(versionStr) != 2 { + return 0, fmt.Errorf("malformatted error, got: %v", versionStr) + } + f, err := strconv.ParseFloat(versionStr[1], 64) if err != nil { return 0, err } @@ -346,7 +348,7 @@ func replaceGoVersionInCodebase(old, new *version.Version, workflowUpdate bool) for _, fileToChange := range filesToChange { err = replaceInFile( - []string{old.String()}, + []*regexp.Regexp{regexp.MustCompile(fmt.Sprintf(`(%s)`, old.String()))}, []string{new.String()}, fileToChange, ) @@ -357,7 +359,7 @@ func replaceGoVersionInCodebase(old, new *version.Version, workflowUpdate bool) if !isSameMajorMinorVersion(old, new) { err = replaceInFile( - []string{fmt.Sprintf("%d.%d", old.Segments()[0], old.Segments()[1])}, + []*regexp.Regexp{regexp.MustCompile(fmt.Sprintf("(%d.%d)", old.Segments()[0], old.Segments()[1]))}, []string{fmt.Sprintf("%d.%d", new.Segments()[0], new.Segments()[1])}, "./go.mod", ) @@ -389,7 +391,10 @@ func updateBootstrapVersionInCodebase(old, new float64, newGoVersion *version.Ve newReplace := fmt.Sprintf("%s%-1g", btv, new) for _, file := range files { err = replaceInFile( - []string{oldReplace, strings.ToUpper(oldReplace)}, + []*regexp.Regexp{ + regexp.MustCompile(fmt.Sprintf(`(%s)`, oldReplace)), + regexp.MustCompile(fmt.Sprintf(`(%s)`, strings.ToUpper(oldReplace))), + }, []string{newReplace, strings.ToUpper(newReplace)}, file, ) @@ -398,10 +403,9 @@ func updateBootstrapVersionInCodebase(old, new float64, newGoVersion *version.Ve } } - oldReplace = fmt.Sprintf("\"bootstrap-version\", \"%-1g\"", old) newReplace = fmt.Sprintf("\"bootstrap-version\", \"%-1g\"", new) err = replaceInFile( - []string{oldReplace}, + []*regexp.Regexp{regexp.MustCompile(`(?i).*bootstrap-version\",[[:space:]]*\"([0-9.]+)\".*`)}, []string{newReplace}, "./test.go", ) @@ -471,8 +475,8 @@ func getListOfFilesInPaths(pathsToExplore []string) ([]string, error) { } // replaceInFile replaces old with new in the given file. -func replaceInFile(old, new []string, fileToChange string) error { - if len(old) != len(new) { +func replaceInFile(oldexps []*regexp.Regexp, new []string, fileToChange string) error { + if len(oldexps) != len(new) { panic("old and new should be of the same length") } @@ -488,8 +492,8 @@ func replaceInFile(old, new []string, fileToChange string) error { } contentStr := string(content) - for i := range old { - contentStr = strings.ReplaceAll(contentStr, old[i], new[i]) + for i, oldex := range oldexps { + contentStr = oldex.ReplaceAllString(contentStr, new[i]) } _, err = f.WriteAt([]byte(contentStr), 0) From 77bd2a73769b115d875f9e4bb1cf61c2065c53be Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 15 Mar 2023 09:29:27 +0200 Subject: [PATCH 27/34] Proper regexp for every string comparison Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index dd4cb9dd00e..49a1d4e2b33 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -58,7 +58,8 @@ var ( It mostly used by the update_golang_version.yml CI workflow that runs on a CRON. `, Run: func(cmd *cobra.Command, args []string) { - _ = cmd.Help() + // _ = cmd.Help() + runUpgradeCmd(cmd, args) }, Args: cobra.NoArgs, } @@ -403,10 +404,9 @@ func updateBootstrapVersionInCodebase(old, new float64, newGoVersion *version.Ve } } - newReplace = fmt.Sprintf("\"bootstrap-version\", \"%-1g\"", new) err = replaceInFile( - []*regexp.Regexp{regexp.MustCompile(`(?i).*bootstrap-version\",[[:space:]]*\"([0-9.]+)\".*`)}, - []string{newReplace}, + []*regexp.Regexp{regexp.MustCompile(`(?i)\"bootstrap-version\",[[:space:]]*\"([0-9.]+)\"`)}, + []string{fmt.Sprintf("\"bootstrap-version\", \"%-1g\"", new)}, "./test.go", ) if err != nil { From e341a1feedd7b031e355dba0fcfff3eb21401b6f Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 15 Mar 2023 10:21:45 +0200 Subject: [PATCH 28/34] use better regex Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 49a1d4e2b33..5ce41329c6c 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -360,8 +360,8 @@ func replaceGoVersionInCodebase(old, new *version.Version, workflowUpdate bool) if !isSameMajorMinorVersion(old, new) { err = replaceInFile( - []*regexp.Regexp{regexp.MustCompile(fmt.Sprintf("(%d.%d)", old.Segments()[0], old.Segments()[1]))}, - []string{fmt.Sprintf("%d.%d", new.Segments()[0], new.Segments()[1])}, + []*regexp.Regexp{regexp.MustCompile(`go[[:space:]]*([0-9.]+)`)}, + []string{fmt.Sprintf("go %d.%d", new.Segments()[0], new.Segments()[1])}, "./go.mod", ) if err != nil { @@ -387,16 +387,10 @@ func updateBootstrapVersionInCodebase(old, new float64, newGoVersion *version.Ve return err } - const btv = "bootstrap_version=" - oldReplace := fmt.Sprintf("%s%-1g", btv, old) - newReplace := fmt.Sprintf("%s%-1g", btv, new) for _, file := range files { err = replaceInFile( - []*regexp.Regexp{ - regexp.MustCompile(fmt.Sprintf(`(%s)`, oldReplace)), - regexp.MustCompile(fmt.Sprintf(`(%s)`, strings.ToUpper(oldReplace))), - }, - []string{newReplace, strings.ToUpper(newReplace)}, + []*regexp.Regexp{regexp.MustCompile(`(?i)ARG[[:space:]]*bootstrap_version[[:space:]]*=[[:space:]]*([0-9.]+)`)}, + []string{fmt.Sprintf("ARG bootstrap_version=%-1g", new)}, file, ) if err != nil { From 89243a7e9dc497c86e6285aba54cc5de68acabd3 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 15 Mar 2023 10:25:51 +0200 Subject: [PATCH 29/34] remove useless flag and improve the auto-PR body description Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 2 ++ go/tools/go-upgrade/go_upgrade.go | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index e31d6ccd1b1..5aeba97fcd9 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -58,6 +58,8 @@ jobs: body: | This Pull Request bumps the Golang version to `go${{steps.detect-and-update.outputs.go-version}}` and the bootstrap version to `${{steps.detect-and-update.outputs.bootstrap-version}}`. + > Do not trust the bot blindly. A thorough code review must be done to ensure all the files have been correctly modified. + There are a few manual steps remaining: - [ ] Make sure you update the Golang version used in the previous and next release branches for the Upgrade/Downgrade tests. - [ ] Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 5ce41329c6c..65a1b4cbcc0 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -47,7 +47,6 @@ var ( workflowUpdate = true allowMajorUpgrade = false isMainBranch = false - goFrom = "" goTo = "" rootCmd = &cobra.Command{ @@ -58,8 +57,7 @@ var ( It mostly used by the update_golang_version.yml CI workflow that runs on a CRON. `, Run: func(cmd *cobra.Command, args []string) { - // _ = cmd.Help() - runUpgradeCmd(cmd, args) + _ = cmd.Help() }, Args: cobra.NoArgs, } From c9579a436c019f8b7da028b4d266c655f8bdbf62 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 15 Mar 2023 10:28:04 +0200 Subject: [PATCH 30/34] add note in the tool usage to run the tool at the root of the repo Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/go_upgrade.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go_upgrade.go index 65a1b4cbcc0..8152be63121 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go_upgrade.go @@ -55,6 +55,8 @@ var ( Long: `go-upgrade allows us to automate some tasks required to bump the version of Golang used throughout our codebase. It mostly used by the update_golang_version.yml CI workflow that runs on a CRON. + +This tool is meant to be run at the root of the repository. `, Run: func(cmd *cobra.Command, args []string) { _ = cmd.Help() From bf35983aa000e17790cd228c23437293a77dd1c7 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Fri, 17 Mar 2023 08:18:39 +0200 Subject: [PATCH 31/34] add comment in .github/workflows/update_golang_version.yml Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 5aeba97fcd9..67c34466c67 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -2,7 +2,7 @@ name: Update Golang Version on: schedule: - - cron: "0 0 * * *" + - cron: "0 0 * * *" # Runs every day at midnight UTC workflow_dispatch: jobs: From c6f56d377aad05628efc41de7c2c0567e6f7ac0d Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Mon, 20 Mar 2023 15:34:57 +0200 Subject: [PATCH 32/34] Apply @mattlord code suggestion Signed-off-by: Florent Poinsard --- go/tools/go-upgrade/{go_upgrade.go => go-upgrade.go} | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) rename go/tools/go-upgrade/{go_upgrade.go => go-upgrade.go} (97%) diff --git a/go/tools/go-upgrade/go_upgrade.go b/go/tools/go-upgrade/go-upgrade.go similarity index 97% rename from go/tools/go-upgrade/go_upgrade.go rename to go/tools/go-upgrade/go-upgrade.go index 8152be63121..ef323bd1149 100644 --- a/go/tools/go-upgrade/go_upgrade.go +++ b/go/tools/go-upgrade/go-upgrade.go @@ -389,8 +389,14 @@ func updateBootstrapVersionInCodebase(old, new float64, newGoVersion *version.Ve for _, file := range files { err = replaceInFile( - []*regexp.Regexp{regexp.MustCompile(`(?i)ARG[[:space:]]*bootstrap_version[[:space:]]*=[[:space:]]*([0-9.]+)`)}, - []string{fmt.Sprintf("ARG bootstrap_version=%-1g", new)}, + []*regexp.Regexp{ + regexp.MustCompile(`(?i)ARG[[:space:]]*bootstrap_version[[:space:]]*=[[:space:]]*[0-9.]+`), // Dockerfile + regexp.MustCompile(`(?i)BOOTSTRAP_VERSION[[:space:]]*=[[:space:]]*[0-9.]+`), // Makefile + }, + []string{ + fmt.Sprintf("ARG bootstrap_version=%-1g", new), // Dockerfile + fmt.Sprintf("BOOTSTRAP_VERSION=%-1g", new), // Makefile + }, file, ) if err != nil { From a5a4327accfdbe284f080b2900d1c77e38e5897b Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 22 Mar 2023 09:34:15 +0200 Subject: [PATCH 33/34] replace usages of old filename Signed-off-by: Florent Poinsard --- .github/workflows/update_golang_version.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 67c34466c67..c7ea3c3a1cc 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -26,13 +26,13 @@ jobs: - name: Detect new version and update codebase id: detect-and-update run: | - old_go_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_go_version) + old_go_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get_go_version) echo "old-go-version=${old_go_version}" >> $GITHUB_OUTPUT if [ ${{ matrix.branch }} == "main" ]; then - go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update --main --allow-major-upgrade + go run ./go/tools/go-upgrade/go-upgrade.go --no-workflow-update --main --allow-major-upgrade else - go run ./go/tools/go-upgrade/go_upgrade.go --no-workflow-update + go run ./go/tools/go-upgrade/go-upgrade.go --no-workflow-update fi output=$(git status -s) @@ -41,8 +41,8 @@ jobs: fi echo "create-pr=true" >> $GITHUB_OUTPUT - go_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_go_version) - bootstrap_version=$(go run ./go/tools/go-upgrade/go_upgrade.go get_bootstrap_version) + go_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get_go_version) + bootstrap_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get_bootstrap_version) echo "go-version=${go_version}" >> $GITHUB_OUTPUT echo "bootstrap-version=${bootstrap_version}" >> $GITHUB_OUTPUT @@ -64,7 +64,7 @@ jobs: - [ ] Make sure you update the Golang version used in the previous and next release branches for the Upgrade/Downgrade tests. - [ ] Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. - [ ] Update the `./.github/workflows/*.yml` files with the newer Golang version, the bot cannot handle that due to permissions. - - To accomplish this, run the following: `go run ./go/tools/go-upgrade/go_upgrade.go --go-from=${{steps.detect-and-update.outputs.old-go-version}} --go-to=${{steps.detect-and-update.outputs.go-version}} update_workflows` + - To accomplish this, run the following: `go run ./go/tools/go-upgrade/go-upgrade.go --go-from=${{steps.detect-and-update.outputs.old-go-version}} --go-to=${{steps.detect-and-update.outputs.go-version}} update_workflows` base: ${{ matrix.branch }} labels: | Skip CI From 0a0e9fa04d5d06e9979a7f20f472ee996093014b Mon Sep 17 00:00:00 2001 From: Florent Poinsard <35779988+frouioui@users.noreply.github.com> Date: Wed, 22 Mar 2023 16:16:56 +0200 Subject: [PATCH 34/34] Apply suggestions from code review Co-authored-by: Matt Lord Signed-off-by: Florent Poinsard <35779988+frouioui@users.noreply.github.com> --- .github/workflows/update_golang_version.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index c7ea3c3a1cc..62f917adfd7 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -26,13 +26,13 @@ jobs: - name: Detect new version and update codebase id: detect-and-update run: | - old_go_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get_go_version) + old_go_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get go-version) echo "old-go-version=${old_go_version}" >> $GITHUB_OUTPUT if [ ${{ matrix.branch }} == "main" ]; then - go run ./go/tools/go-upgrade/go-upgrade.go --no-workflow-update --main --allow-major-upgrade + go run ./go/tools/go-upgrade/go-upgrade.go upgrade --workflow-update=false --main --allow-major-upgrade else - go run ./go/tools/go-upgrade/go-upgrade.go --no-workflow-update + go run ./go/tools/go-upgrade/go-upgrade.go upgrade --workflow-update=false fi output=$(git status -s) @@ -41,8 +41,8 @@ jobs: fi echo "create-pr=true" >> $GITHUB_OUTPUT - go_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get_go_version) - bootstrap_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get_bootstrap_version) + go_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get go-version) + bootstrap_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get bootstrap-version) echo "go-version=${go_version}" >> $GITHUB_OUTPUT echo "bootstrap-version=${bootstrap_version}" >> $GITHUB_OUTPUT @@ -64,7 +64,7 @@ jobs: - [ ] Make sure you update the Golang version used in the previous and next release branches for the Upgrade/Downgrade tests. - [ ] Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. - [ ] Update the `./.github/workflows/*.yml` files with the newer Golang version, the bot cannot handle that due to permissions. - - To accomplish this, run the following: `go run ./go/tools/go-upgrade/go-upgrade.go --go-from=${{steps.detect-and-update.outputs.old-go-version}} --go-to=${{steps.detect-and-update.outputs.go-version}} update_workflows` + - To accomplish this, run the following: `go run ./go/tools/go-upgrade/go-upgrade.go upgrade workflows --go-to=${{steps.detect-and-update.outputs.go-version}}` base: ${{ matrix.branch }} labels: | Skip CI