Skip to content

Commit

Permalink
feat(internal/postprocessor): only regen snippets for changed modules (…
Browse files Browse the repository at this point in the history 
…#7300)

This change identifies the changed modules in the PR using the commit URLs in the open OwlBot PR description. The scopes are used to limit snippet regeneration to only those modules.
  • Loading branch information
@adrianajg
adrianajg authored Feb 8, 2023
1 parent 935ba3f commit 220f8a5
Showing 1 changed file with 107 additions and 35 deletions.
142 changes: 107 additions & 35 deletions internal/postprocessor/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,12 @@ func main() {
log.Println("googleapis-dir set to", *googleapisDir)
log.Println("branch set to", *branchOverride)
log.Println("prFilepath is", *prFilepath)
log.Println("directories are", *directories)

var modules []string
dirSlice := []string{}
if *directories != "" {
dirSlice := strings.Split(*directories, ",")
for _, dir := range dirSlice {
modules = append(modules, filepath.Join(*clientRoot, dir))
}
log.Println("Postprocessor running on", modules)
log.Println("Postprocessor running on", dirSlice)
} else {
log.Println("Postprocessor running on all modules.")
}
Expand All @@ -103,11 +101,13 @@ func main() {
c := &config{
googleapisDir: *googleapisDir,
googleCloudDir: *clientRoot,
modules: modules,
modules: dirSlice,
branchOverride: *branchOverride,
githubUsername: *githubUsername,
prFilepath: *prFilepath,
runAll: runAll,
prTitle: "",
prBody: "",
}

if err := c.run(ctx); err != nil {
Expand All @@ -120,11 +120,19 @@ func main() {
type config struct {
googleapisDir string
googleCloudDir string
modules []string

// At this time modules are either provided at the time of invocation locally
// and extracted from the open OwlBot PR description. If we would like
// the postprocessor to be able to be run on non-OwlBot PRs, we would
// need to change the method of populating this field.
modules []string

branchOverride string
githubUsername string
prFilepath string
runAll bool
prTitle string
prBody string
}

// runAll uses git to tell if the PR being updated should run all post
Expand All @@ -150,7 +158,16 @@ func (c *config) run(ctx context.Context) error {
log.Println("exiting post processing early")
return nil
}
if err := gocmd.ModTidyAll(c.googleCloudDir); err != nil {
// TODO(guadriana): Once the PR for initializing new modules is merged, we
// will need to add logic here that sets c.modules to modConfigs (a slice of
// all modules) if branchOverride != ""

// TODO(codyoss): In the future we may want to make it possible to be able
// to run this locally with a user defined remote branch.
if err := c.SetScopesAndPRInfo(ctx); err != nil {
return err
}
if err := c.TidyAffectedMods(); err != nil {
return err
}
if err := gocmd.Vet(c.googleCloudDir); err != nil {
Expand All @@ -162,28 +179,44 @@ func (c *config) run(ctx context.Context) error {
if _, err := c.Manifest(generator.MicrogenGapicConfigs); err != nil {
return err
}
// TODO(codyoss): In the future we may want to make it possible to be able
// to run this locally with a user defined remote branch.
if err := c.AmendPRDescription(ctx); err != nil {
if err := c.WritePRInfoToFile(); err != nil {
return err
}
return nil
}

func (c *config) getDirs() []string {
dirs := []string{}
for _, module := range c.modules {
dirs = append(dirs, filepath.Join(c.googleCloudDir, module))
}
return dirs
}

func (c *config) TidyAffectedMods() error {
dirs := c.getDirs()
for _, dir := range dirs {
if err := gocmd.ModTidy(dir); err != nil {
return err
}
}
return nil
}

// RegenSnippets regenerates the snippets for all GAPICs configured to be generated.
func (c *config) RegenSnippets() error {
log.Println("regenerating snippets")

snippetDir := filepath.Join(c.googleCloudDir, "internal", "generated", "snippets")
apiShortnames, err := generator.ParseAPIShortnames(c.googleapisDir, generator.MicrogenGapicConfigs, generator.ManualEntries)

confs := c.getChangedClientConfs()
apiShortnames, err := generator.ParseAPIShortnames(c.googleapisDir, confs, generator.ManualEntries)
if err != nil {
return err
}
if err := gensnippets.GenerateSnippetsDirs(c.googleCloudDir, snippetDir, apiShortnames, c.modules); err != nil {
dirs := c.getDirs()
if err := gensnippets.GenerateSnippetsDirs(c.googleCloudDir, snippetDir, apiShortnames, dirs); err != nil {
log.Printf("warning: got the following non-fatal errors generating snippets: %v", err)
}
if err := c.replaceAllForSnippets(c.googleCloudDir, snippetDir); err != nil {
if err := c.replaceAllForSnippets(snippetDir); err != nil {
return err
}
if err := gocmd.ModTidy(snippetDir); err != nil {
Expand All @@ -193,15 +226,44 @@ func (c *config) RegenSnippets() error {
return nil
}

func (c *config) replaceAllForSnippets(googleCloudDir, snippetDir string) error {
return execv.ForEachMod(googleCloudDir, func(dir string) error {
// getChangedClientConfs iterates through the MicrogenGapicConfigs and returns
// a slice of the entries corresponding to modified modules and clients
func (c *config) getChangedClientConfs() []*generator.MicrogenConfig {
if len(c.modules) != 0 {
runConfs := []*generator.MicrogenConfig{}
for _, conf := range generator.MicrogenGapicConfigs {
for _, scope := range c.modules {
scopePathElement := "/" + scope + "/"
if strings.Contains(conf.InputDirectoryPath, scopePathElement) {
runConfs = append(runConfs, conf)
}
}
}
return runConfs
}
return generator.MicrogenGapicConfigs
}

func (c *config) replaceAllForSnippets(snippetDir string) error {
return execv.ForEachMod(c.googleCloudDir, func(dir string) error {
processMod := false
if c.modules != nil {
// Checking each path component in its entirety prevents mistaken addition of modules whose names
// contain the scope as a substring. For example if the scope is "video" we do not want to regenerate
// snippets for "videointelligence"
dirSlice := strings.Split(dir, "/")
for _, mod := range c.modules {
if !strings.Contains(dir, mod) {
return nil
for _, dirElem := range dirSlice {
if mod == dirElem {
processMod = true
break
}
}
}
}
if !processMod {
return nil
}
if dir == snippetDir {
return nil
}
Expand Down Expand Up @@ -280,7 +342,7 @@ func docURL(cloudDir, importPath string) (string, error) {
return "https://cloud.google.com/go/docs/reference/" + mod + "/latest/" + pkgPath, nil
}

func (c *config) AmendPRDescription(ctx context.Context) error {
func (c *config) SetScopesAndPRInfo(ctx context.Context) error {
log.Println("Amending PR title and body")
pr, err := c.getPR(ctx)
if err != nil {
Expand All @@ -290,13 +352,16 @@ func (c *config) AmendPRDescription(ctx context.Context) error {
if err != nil {
return err
}
return c.writePRCommitToFile(newPRTitle, newPRBody)
c.prTitle = newPRTitle
c.prBody = newPRBody
return nil
}

func (c *config) processCommit(title, body string) (string, string, error) {
var newPRTitle string
var commitTitle string
var commitTitleIndex int
var modules []string

bodySlice := strings.Split(body, "\n")
for index, line := range bodySlice {
Expand All @@ -312,7 +377,12 @@ func (c *config) processCommit(title, body string) (string, string, error) {
continue
}
hash := extractHashFromLine(line)
scope, err := c.getScopeFromGoogleapisCommitHash(hash)
scopes, err := c.getScopesFromGoogleapisCommitHash(hash)
modules = append(modules, scopes...)
var scope string
if len(scopes) == 1 {
scope = scopes[0]
}
if err != nil {
return "", "", err
}
Expand All @@ -324,6 +394,7 @@ func (c *config) processCommit(title, body string) (string, string, error) {
bodySlice[commitTitleIndex] = newCommitTitle
}
body = strings.Join(bodySlice, "\n")
c.modules = append(c.modules, modules...)
return newPRTitle, body, nil
}

Expand All @@ -349,14 +420,14 @@ func (c *config) getPR(ctx context.Context) (*github.PullRequest, error) {
return owlbotPR, nil
}

func (c *config) getScopeFromGoogleapisCommitHash(commitHash string) (string, error) {
func (c *config) getScopesFromGoogleapisCommitHash(commitHash string) ([]string, error) {
files, err := c.filesChanged(commitHash)
if err != nil {
return "", err
return nil, err
}
// if no files changed, return empty string
if len(files) == 0 {
return "", nil
return nil, nil
}
scopesMap := make(map[string]bool)
scopes := []string{}
Expand All @@ -375,12 +446,7 @@ func (c *config) getScopeFromGoogleapisCommitHash(commitHash string) (string, er
}
}
}
// if no in-scope packages are found or if many packages found, return empty string
if len(scopes) != 1 {
return "", nil
}
// if single scope found, return
return scopes[0], nil
return scopes, nil
}

// filesChanged returns a list of files changed in a commit for the provdied
Expand Down Expand Up @@ -421,8 +487,9 @@ func updateCommitTitle(title, titlePkg string) string {
return newTitle
}

// writePRCommitToFile uses OwlBot env variable specified path to write updated PR title and body at that location
func (c *config) writePRCommitToFile(title, body string) error {
// WritePRInfoToFile uses OwlBot env variable specified path to write updated
// PR title and body at that location
func (c *config) WritePRInfoToFile() error {
// if file exists at location, delete
if err := os.Remove(c.prFilepath); err != nil {
if errors.Is(err, fs.ErrNotExist) {
Expand All @@ -436,7 +503,12 @@ func (c *config) writePRCommitToFile(title, body string) error {
return err
}
defer f.Close()
if _, err := f.WriteString(fmt.Sprintf("%s\n\n%s", title, body)); err != nil {
if c.prTitle == "" && c.prBody == "" {
log.Println("No updated PR info found, will not write PR title and description to file.")
return nil
}
log.Println("Writing PR title and description to file.")
if _, err := f.WriteString(fmt.Sprintf("%s\n\n%s", c.prTitle, c.prBody)); err != nil {
return err
}
return nil
Expand Down

0 comments on commit 220f8a5

Please sign in to comment.