From 59ce02c13f265741a8f1f0f7ad5109bf83e3df82 Mon Sep 17 00:00:00 2001 From: Adriana <43220284+adrianajg@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:06:11 -0500 Subject: [PATCH] feat(internal/postprocessor): detect and initialize new modules (#7288) Detects new clients and modules and generates minimum required files for each. New client generated files: - `version.go` New module generated files: - `internal/version.go` - `go.mod` - `README.md` - `CHANGES.md` --- internal/postprocessor/README.md | 17 +- internal/postprocessor/_README.md.txt | 44 ++++ .../postprocessor/_internal_version.go.txt | 20 ++ internal/postprocessor/_version.go.txt | 23 ++ internal/postprocessor/main.go | 196 +++++++++++++++++- internal/postprocessor/modconfig.go | 144 +++++++++++++ 6 files changed, 427 insertions(+), 17 deletions(-) create mode 100644 internal/postprocessor/_README.md.txt create mode 100644 internal/postprocessor/_internal_version.go.txt create mode 100644 internal/postprocessor/_version.go.txt create mode 100644 internal/postprocessor/modconfig.go diff --git a/internal/postprocessor/README.md b/internal/postprocessor/README.md index c81dc49dbdf8..eacc8f8a02b6 100644 --- a/internal/postprocessor/README.md +++ b/internal/postprocessor/README.md @@ -37,20 +37,17 @@ clients in the root directory. ### Run post-processor on all clients -From the `google-cloud-go/internal/postprocessor` directory run: - +From the `google-cloud-go/internal/postprocessor` directory run: ```bash -go run main.go -stage-dir="../../owl-bot-staging/src/" -client-root="../.." -googleapis-dir="/path/to/local/googleapis" +go run main.go -client-root="../.." -googleapis-dir="/path/to/local/googleapis" ``` ### Run post-processor on select clients -From the `google-cloud-go/internal/postprocessor` directory run the same -command, but with an added `dirs` flag containing a comma-separated list of the -names of the clients on which to run the post-processor. The example below shows -the command for running the post-processor on the `accessapproval` and `asset` -libraries: - +From the `google-cloud-go/internal/postprocessor` directory run the same command, but with an added `dirs` flag containing a comma-separated list of the names of the clients on which to run the post-processor. The example below shows the command for running the post-processor on the `accessapproval` and `asset` libraries: ```bash -go run main.go -stage-dir="../../owl-bot-staging/src/" -client-root="../.." -googleapis-dir="/path/to/local/googleapis" -dirs="accessapproval,asset" +go run main.go -client-root="../.." -googleapis-dir="/path/to/local/googleapis" -dirs="accessapproval,asset" ``` + +### Initializing new modules +To initialize the `internal/version.go`, `go.mod`, `README.md`, and `CHANGES.md` files in a new module, add the module to the slice in `modconfig.go`. The entry should correspond to the location where the `go.mod` file should be initialized minus the prefix "google-cloud-go/" diff --git a/internal/postprocessor/_README.md.txt b/internal/postprocessor/_README.md.txt new file mode 100644 index 000000000000..cc5b9cabf113 --- /dev/null +++ b/internal/postprocessor/_README.md.txt @@ -0,0 +1,44 @@ +# {{.Name}} + +[![Go Reference](https://pkg.go.dev/badge/{{.ImportPath}}.svg)](https://pkg.go.dev/{{.ImportPath}}) + +Go Client Library for {{.Name}}. + +## Install + +```bash +go get {{.ImportPath}} +``` + +## Stability + +The stability of this module is indicated by SemVer. + +However, a `v1+` module may have breaking changes in two scenarios: + +* Packages with `alpha` or `beta` in the import path +* The GoDoc has an explicit stability disclaimer (for example, for an experimental feature). + +## Google Cloud Samples + +To browse ready to use code samples check [Google Cloud Samples](https://cloud.google.com/docs/samples?l=go). + +## Go Version Support + +See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported) +section in the root directory's README. + +## Authorization + +See the [Authorization](https://github.com/googleapis/google-cloud-go#authorization) +section in the root directory's README. + +## Contributing + +Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md) +document for details. + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. See +[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct) +for more information. diff --git a/internal/postprocessor/_internal_version.go.txt b/internal/postprocessor/_internal_version.go.txt new file mode 100644 index 000000000000..cf3aa6a12a70 --- /dev/null +++ b/internal/postprocessor/_internal_version.go.txt @@ -0,0 +1,20 @@ +// Copyright {{.Year}} Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by gapicgen. DO NOT EDIT. + +package internal + +// Version is the current tagged release of the library. +const Version = "0.0.0" diff --git a/internal/postprocessor/_version.go.txt b/internal/postprocessor/_version.go.txt new file mode 100644 index 000000000000..e97789af3eff --- /dev/null +++ b/internal/postprocessor/_version.go.txt @@ -0,0 +1,23 @@ +// Copyright {{.Year}} Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by gapicgen. DO NOT EDIT. + +package {{.Package}} + +import "{{.ModuleRootInternal}}" + +func init() { + versionClient = internal.Version +} \ No newline at end of file diff --git a/internal/postprocessor/main.go b/internal/postprocessor/main.go index 4af3d23fc313..6eb0ccf295fb 100644 --- a/internal/postprocessor/main.go +++ b/internal/postprocessor/main.go @@ -16,16 +16,19 @@ package main import ( "context" + _ "embed" "encoding/json" "errors" "flag" "fmt" + "html/template" "io/fs" "log" "os" "path/filepath" "regexp" "strings" + "time" "cloud.google.com/go/internal/gapicgen/generator" "cloud.google.com/go/internal/gapicgen/git" @@ -51,6 +54,15 @@ var ( secondPartTitlePattern = regexp.MustCompile(`.*\: *\` + apiNameOwlBotScope + ` *(?P.*)`) ) +var ( + //go:embed _README.md.txt + readmeTmpl string + //go:embed _version.go.txt + versionTmpl string + //go:embed _internal_version.go.txt + internalVersionTmpl string +) + func main() { clientRoot := flag.String("client-root", "/workspace/google-cloud-go", "Path to clients.") googleapisDir := flag.String("googleapis-dir", "", "Path to googleapis/googleapis repo.") @@ -158,15 +170,17 @@ func (c *config) run(ctx context.Context) error { log.Println("exiting post processing early") return nil } - // TODO(guadriana): Once the PR for initializing new modules is merged, we - // will need to add logic here that sets c.modules to modConfigs (a slice of - // all modules) if branchOverride != "" - - // TODO(codyoss): In the future we may want to make it possible to be able - // to run this locally with a user defined remote branch. + manifest, err := c.Manifest(generator.MicrogenGapicConfigs) + if err != nil { + return err + } + if err := c.InitializeNewModules(manifest); err != nil { + return err + } if err := c.SetScopesAndPRInfo(ctx); err != nil { return err } + if err := c.TidyAffectedMods(); err != nil { return err } @@ -179,12 +193,143 @@ func (c *config) run(ctx context.Context) error { if _, err := c.Manifest(generator.MicrogenGapicConfigs); err != nil { return err } + // TODO(codyoss): In the future we may want to make it possible to be able + // to run this locally with a user defined remote branch. if err := c.WritePRInfoToFile(); err != nil { return err } return nil } +// InitializeNewModule detects new modules and clients and generates the required minimum files +// For modules, the minimum required files are internal/version.go, README.md, CHANGES.md, and go.mod +// For clients, the minimum required files are a version.go file +func (c *config) InitializeNewModules(manifest map[string]generator.ManifestEntry) error { + log.Println("checking for new modules and clients") + for _, moduleName := range moduleConfigs { + modulePath := filepath.Join(c.googleCloudDir, moduleName) + importPath := filepath.Join("cloud.google.com/go", moduleName) + + pathToModVersionFile := filepath.Join(modulePath, "internal/version.go") + // Check if /internal/version.go file exists + if _, err := os.Stat(pathToModVersionFile); errors.Is(err, fs.ErrNotExist) { + var serviceImportPath string + for _, conf := range generator.MicrogenGapicConfigs { + if strings.Contains(conf.ImportPath, importPath) { + serviceImportPath = conf.ImportPath + break + } + } + if serviceImportPath == "" { + return fmt.Errorf("no corresponding config found for module %s. Cannot generate min required files", moduleName) + } + // serviceImportPath here should be a valid ImportPath from a MicrogenGapicConfigs + apiName := manifest[serviceImportPath].Description + if err := c.generateMinReqFilesNewMod(moduleName, modulePath, importPath, apiName); err != nil { + return err + } + } + // Check if version.go files exist for each client + filepath.WalkDir(modulePath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + return nil + } + splitPath := strings.Split(path, "/") + lastElement := splitPath[len(splitPath)-1] + if !strings.Contains(lastElement, "apiv") { + return nil + } + pathToClientVersionFile := filepath.Join(path, "version.go") + if _, err = os.Stat(pathToClientVersionFile); errors.Is(err, fs.ErrNotExist) { + log.Println("generating version.go file in", path) + if err := c.generateVersionFile(moduleName, path); err != nil { + return err + } + } + return nil + }) + } + return nil +} + +func (c *config) generateMinReqFilesNewMod(moduleName, modulePath, importPath, apiName string) error { + log.Println("generating files for new module", apiName) + if err := generateReadmeAndChanges(modulePath, importPath, apiName); err != nil { + return err + } + if err := c.generateInternalVersionFile(moduleName); err != nil { + return err + } + if err := c.generateModule(modulePath, importPath); err != nil { + return err + } + return nil +} + +func (c *config) generateModule(modPath, importPath string) error { + if err := os.MkdirAll(modPath, os.ModePerm); err != nil { + return err + } + log.Printf("Creating %s/go.mod", modPath) + return gocmd.ModInit(modPath, importPath) +} + +func (c *config) generateVersionFile(moduleName, modulePath string) error { + // These directories are not modules on purpose, don't generate a version + // file for them. + if strings.Contains(modulePath, "debugger/apiv2") { + return nil + } + rootPackage := filepath.Dir(modulePath) + rootModInternal := fmt.Sprintf("cloud.google.com/go/%s/internal", rootPackage) + + f, err := os.Create(filepath.Join(modulePath, "version.go")) + if err != nil { + return err + } + defer f.Close() + + t := template.Must(template.New("version").Parse(versionTmpl)) + versionData := struct { + Year int + Package string + ModuleRootInternal string + }{ + Year: time.Now().Year(), + Package: moduleName, + ModuleRootInternal: rootModInternal, + } + if err := t.Execute(f, versionData); err != nil { + return err + } + return nil +} + +func (c *config) generateInternalVersionFile(apiName string) error { + rootModInternal := filepath.Join(apiName, "internal") + os.MkdirAll(filepath.Join(c.googleCloudDir, rootModInternal), os.ModePerm) + + f, err := os.Create(filepath.Join(c.googleCloudDir, rootModInternal, "version.go")) + if err != nil { + return err + } + defer f.Close() + + t := template.Must(template.New("internal_version").Parse(internalVersionTmpl)) + internalVersionData := struct { + Year int + }{ + Year: time.Now().Year(), + } + if err := t.Execute(f, internalVersionData); err != nil { + return err + } + return nil +} + func (c *config) getDirs() []string { dirs := []string{} for _, module := range c.modules { @@ -203,6 +348,38 @@ func (c *config) TidyAffectedMods() error { return nil } +// Copied from generator package +func generateReadmeAndChanges(path, importPath, apiName string) error { + readmePath := filepath.Join(path, "README.md") + log.Printf("Creating %q", readmePath) + readmeFile, err := os.Create(readmePath) + if err != nil { + return err + } + defer readmeFile.Close() + t := template.Must(template.New("readme").Parse(readmeTmpl)) + readmeData := struct { + Name string + ImportPath string + }{ + Name: apiName, + ImportPath: importPath, + } + if err := t.Execute(readmeFile, readmeData); err != nil { + return err + } + + changesPath := filepath.Join(path, "CHANGES.md") + log.Printf("Creating %q", changesPath) + changesFile, err := os.Create(changesPath) + if err != nil { + return err + } + defer changesFile.Close() + _, err = changesFile.WriteString("# Changes\n") + return err +} + // RegenSnippets regenerates the snippets for all GAPICs configured to be generated. func (c *config) RegenSnippets() error { log.Println("regenerating snippets") @@ -394,7 +571,12 @@ func (c *config) processCommit(title, body string) (string, string, error) { bodySlice[commitTitleIndex] = newCommitTitle } body = strings.Join(bodySlice, "\n") - c.modules = append(c.modules, modules...) + if c.branchOverride != "" { + c.modules = []string{} + c.modules = append(c.modules, moduleConfigs...) + } else { + c.modules = append(c.modules, modules...) + } return newPRTitle, body, nil } diff --git a/internal/postprocessor/modconfig.go b/internal/postprocessor/modconfig.go new file mode 100644 index 000000000000..82981bf7fb7b --- /dev/null +++ b/internal/postprocessor/modconfig.go @@ -0,0 +1,144 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// ModuleConfig represents a module. +var moduleConfigs = []string{ + "accessapproval", + "accesscontextmanager", + "aiplatform", + "analytics", + "apigateway", + "apigeeconnect", + "apigeeregistry", + "apikeys", + "appengine", + "area120", + "artifactregistry", + "asset", + "assuredworkloads", + "automl", + "baremetalsolution", + "batch", + "beyondcorp", + "bigquery", + "bigtable", + "billing", + "binaryauthorization", + "certificatemanager", + "channel", + "cloudbuild", + "clouddms", + "cloudtasks", + "compute", + "contactcenterinsights", + "container", + "containeranalysis", + "datacatalog", + "dataflow", + "dataform", + "datafusion", + "datalabeling", + "dataplex", + "dataproc", + "dataqna", + "datastore", + "datastream", + "deploy", + "dialogflow", + "dlp", + "documentai", + "domains", + "edgecontainer", + "errorreporting", + "essentialcontacts", + "eventarc", + "filestore", + "firestore", + "functions", + "gaming", + "gkebackup", + "gkeconnect", + "gkehub", + "gkemulticloud", + "grafeas", + "gsuiteaddons", + "iam", + "iap", + "ids", + "iot", + "kms", + "language", + "lifesciences", + "logging", + "longrunning", + "managedidentities", + "maps", + "mediatranslation", + "memcache", + "metastore", + "monitoring", + "networkconnectivity", + "networkmanagement", + "networksecurity", + "notebooks", + "optimization", + "orchestration", + "orgpolicy", + "osconfig", + "oslogin", + "phishingprotection", + "policytroubleshooter", + "privatecatalog", + "profiler", + "pubsub", + "pubsublite", + "recaptchaenterprise", + "recaptchaenterprise/v2", + "recommendationengine", + "recommender", + "redis", + "resourcemanager", + "resourcesettings", + "retail", + "run", + "scheduler", + "secretmanager", + "security", + "securitycenter", + "servicecontrol", + "servicedirectory", + "servicemanagement", + "serviceusage", + "shell", + "spanner", + "speech", + "storagetransfer", + "talent", + "texttospeech", + "tpu", + "trace", + "translate", + "video", + "videointelligence", + "vision", + "vision/v2", + "vmmigration", + "vmwareengine", + "vpcaccess", + "webrisk", + "websecurityscanner", + "workflows", +}