chore: refactor genlocal into genbot (#3674)

Previously if you wanted to run genlocal you needed to have a
bunch of dependencies installed. Some of which can be a little
tricky to install correctly. These dependencies happen to be the
same as genbot. For this reason this commit refactors genlocal
into genbot so that the genbot in local mode can be run from
a docker container that has the exact same deps installed as our
nightly regen.

internal/gapicgen/cmd/genbot/Dockerfile

@@ -41,6 +41,11 @@ RUN mkdir /root/.ssh && echo -e "Host github.com\n\tStrictHostKeyChecking no\n"
 
 RUN echo -e '#!/bin/bash\n\
     set -ex\n\
+    if [[ ${GENBOT_LOCAL_MODE} = "true" ]]; then\n\
+    cd internal/gapicgen\n\
+    go run cloud.google.com/go/internal/gapicgen/cmd/genbot\n\
+    exit 0\n\
+    fi\n\
     go mod download \n\
     go run cloud.google.com/go/internal/gapicgen/cmd/genbot \
     ' >> /run.sh

internal/gapicgen/cmd/genbot/README.md

@@ -3,23 +3,38 @@
 genbot is a binary for generating gapics and creating PRs with the results.
 It is intended to be used as a bot, though it can be run locally too.
 
-## Getting certs
-
-### Github
+## Prerequisites for running locally
+
+Note that only step one, listed below, is required if you plan to run the code
+in docker.
+
+1. Clone this project: `git clone https://github.com/googleapis/google-cloud-go.git`
+1. Install [protoc](https://github.com/protocolbuffers/protobuf/releases)
+1. Install [Go](http://golang.org/dl)
+1. Add `$GOPATH/bin` to `PATH`
+1. Create a [GitHub access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line).
+1. Install Go tools:
+
+```bash
+go get \
+    github.com/golang/protobuf/protoc-gen-go \
+    golang.org/x/lint/golint \
+    golang.org/x/tools/cmd/goimports \
+    honnef.co/go/tools/cmd/staticcheck \
+    github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic
+```
 
-For Github, you need to generate/supply a Personal Access Token.  More
-information on how that's done can be found here:
-[creating a personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line).
+## Generating code and PRs(bot mode)
 
-## Running locally
+### Run genbot locally
 
 Note: this may change your `~/.gitconfig`, `~/.gitcookies`, and use up
 non-trivial amounts of space on your computer.
 
 1. Make sure you are on a non-Windows platform. If you are using windows
    continue on to the docker instructions.
-2. Make sure you have all the tools installed listed in genlocal's README.md
-3. Run:
+1. Make sure you have all the tools installed listed in genlocal's README.md
+1. Run:
 
 ```shell
 cd /path/to/internal/gapicgen
@@ -30,7 +45,7 @@ go run cloud.google.com/go/internal/gapicgen/cmd/genbot \
     [email protected] \
 ```
 
-## Run with docker
+### Run genbot with docker
 
 Note: this can be quite slow (~10m).
 
@@ -41,12 +56,51 @@ Note: this may leave a lot of docker resources laying around. Use
 cd /path/to/internal/gapicgen/cmd/genbot
 docker build . -t genbot
 docker run -t --rm --privileged \
-    -v `pwd`/../..:/gapicgen \
-    -e GITHUB_ACCESS_TOKEN \
-    -e GITHUB_USERNAME \
-    -e GITHUB_NAME \
-    -e GITHUB_EMAIL \
-    genbot
+   -v `pwd`/../..:/gapicgen \
+   -e GITHUB_ACCESS_TOKEN \
+   -e GITHUB_USERNAME \
+   -e GITHUB_NAME \
+   -e GITHUB_EMAIL \
+   genbot
+```
+
+## Generating code (local mode)
+
+Sometimes you may want to just generate gapic sources to test out
+new changes, test the generation of a new library, test new generator tweaks,
+run generators against googleapis-private, and various other local tasks. The
+local mode in genbot allows you to do just that.
+
+### Run genbot(local mode) locally
+
+```shell
+cd /path/to/internal/gapicgen
+go run cloud.google.com/go/internal/gapicgen/cmd/genbot \
+   -local \
+   -only-gapics \
+   -gocloud-dir=/path/to/google-cloud-go \
+   -gapic=cloud.google.com/go/foo/apiv1
+```
+
+### Run genbot(local mode) with docker
+
+```shell
+cd /path/to/internal/gapicgen
+docker build -t genbot -f cmd/genbot/Dockerfile .
+docker run --rm \
+   -v `pwd`/../..:/gapicgen \
+   -e GENBOT_LOCAL_MODE=true \
+   -e ONLY_GAPICS=true \
+   -e GOCLOUD_DIR=/gapicgen \
+   -e GAPIC_TO_GENERATE=cloud.google.com/go/foo/apiv1 \
+   genbot \
+```
+
+Note you can optionally mount in your Go module cache if you have Go installed.
+This will speed up the build a bit:
+
+```shell
+-v `go env GOMODCACHE`:/root/go/pkg/mod
 ```
 
 ## FAQ

internal/gapicgen/cmd/genbot/bot.go

@@ -0,0 +1,83 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"time"
+)
+
+func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName, githubEmail string) error {
+	for k, v := range map[string]string{
+		"githubAccessToken": githubAccessToken,
+		"githubUsername":    githubUsername,
+		"githubName":        githubName,
+		"githubEmail":       githubEmail,
+	} {
+		if v == "" {
+			log.Printf("missing or empty value for required flag --%s\n", k)
+			flag.PrintDefaults()
+		}
+	}
+
+	// Setup the client and git environment.
+	githubClient, err := NewGithubClient(ctx, githubUsername, githubName, githubEmail, githubAccessToken)
+	if err != nil {
+		return err
+	}
+
+	// Check current regen status.
+	if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "open"); err != nil {
+		return err
+	} else if pr != nil {
+		return fmt.Errorf("There is already a re-generation in progress")
+	}
+	if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "open"); err != nil {
+		return err
+	} else if pr != nil {
+		if err := updateGocloudPR(ctx, githubClient, pr); err != nil {
+			return err
+		}
+		return nil
+	}
+	log.Println("checking if a pull request was already opened and merged today")
+	if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "closed"); err != nil {
+		return err
+	} else if pr != nil && hasCreatedPRToday(pr.Created) {
+		log.Println("skipping generation, already created and merged a go-genproto PR today")
+		return nil
+	}
+	if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "closed"); err != nil {
+		return err
+	} else if pr != nil && hasCreatedPRToday(pr.Created) {
+		log.Println("skipping generation, already created and merged a google-cloud-go PR today")
+		return nil
+	}
+
+	return generate(ctx, githubClient)
+}
+
+// hasCreatedPRToday checks if the created time of a PR is from today.
+func hasCreatedPRToday(created time.Time) bool {
+	now := time.Now()
+	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
+	log.Printf("Times -- Now: %v\tToday: %v\tPR Created: %v", now, today, created)
+	return created.After(today)
+}

internal/gapicgen/cmd/genbot/generate.go

@@ -53,16 +53,16 @@ func generate(ctx context.Context, githubClient *GithubClient) error {
 
 	grp, _ := errgroup.WithContext(ctx)
 	grp.Go(func() error {
-		return gitClone("https://github.com/googleapis/googleapis", googleapisDir)
+		return gitDeepClone("https://github.com/googleapis/googleapis", googleapisDir)
 	})
 	grp.Go(func() error {
-		return gitClone("https://github.com/googleapis/go-genproto", genprotoDir)
+		return gitDeepClone("https://github.com/googleapis/go-genproto", genprotoDir)
 	})
 	grp.Go(func() error {
-		return gitClone("https://github.com/googleapis/google-cloud-go", gocloudDir)
+		return gitDeepClone("https://github.com/googleapis/google-cloud-go", gocloudDir)
 	})
 	grp.Go(func() error {
-		return gitClone("https://github.com/protocolbuffers/protobuf", protoDir)
+		return gitDeepClone("https://github.com/protocolbuffers/protobuf", protoDir)
 	})
 	if err := grp.Wait(); err != nil {
 		log.Println(err)
@@ -140,7 +140,7 @@ func generate(ctx context.Context, githubClient *GithubClient) error {
 }
 
 // gitClone clones a repository in the given directory.
-func gitClone(repo, dir string) error {
+func gitDeepClone(repo, dir string) error {
 	log.Printf("cloning %s\n", repo)
 
 	_, err := git.PlainClone(dir, false, &git.CloneOptions{

internal/gapicgen/cmd/genbot/github.go

@@ -235,7 +235,7 @@ git push origin $BRANCH_NAME
 	// Can't assign the submitter of the PR as a reviewer.
 	var reviewers []string
 	for _, r := range githubReviewers {
-		if r != *githubUsername {
+		if r != gc.Username {
 			reviewers = append(reviewers, r)
 		}
 	}

internal/gapicgen/cmd/genbot/local.go

@@ -0,0 +1,104 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package main
+
+import (
+	"context"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	"cloud.google.com/go/internal/gapicgen/generator"
+	"golang.org/x/sync/errgroup"
+	"gopkg.in/src-d/go-git.v4"
+)
+
+type localConfig struct {
+	googleapisDir   string
+	gocloudDir      string
+	genprotoDir     string
+	protoDir        string
+	gapicToGenerate string
+	onlyGapics      bool
+}
+
+func genLocal(ctx context.Context, c localConfig) error {
+	log.Println("creating temp dir")
+	tmpDir, err := ioutil.TempDir("", "update-genproto")
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Printf("temp dir created at %s\n", tmpDir)
+	tmpGoogleapisDir := filepath.Join(tmpDir, "googleapis")
+	tmpGenprotoDir := filepath.Join(tmpDir, "genproto")
+	tmpGocloudDir := filepath.Join(tmpDir, "gocloud")
+	tmpProtoDir := filepath.Join(tmpDir, "proto")
+
+	// Clone repositories if needed.
+	grp, _ := errgroup.WithContext(ctx)
+	gitShallowClone(grp, "https://github.com/googleapis/googleapis.git", c.googleapisDir, tmpGoogleapisDir)
+	gitShallowClone(grp, "https://github.com/googleapis/go-genproto", c.genprotoDir, tmpGenprotoDir)
+	gitShallowClone(grp, "https://github.com/googleapis/google-cloud-go", c.gocloudDir, tmpGocloudDir)
+	gitShallowClone(grp, "https://github.com/protocolbuffers/protobuf", c.protoDir, tmpProtoDir)
+	if err := grp.Wait(); err != nil {
+		log.Println(err)
+	}
+
+	// Regen.
+	conf := &generator.Config{
+		GoogleapisDir:     deafultDir(tmpGoogleapisDir, c.googleapisDir),
+		GenprotoDir:       deafultDir(tmpGenprotoDir, c.genprotoDir),
+		GapicDir:          deafultDir(tmpGocloudDir, c.gocloudDir),
+		ProtoDir:          deafultDir(tmpProtoDir, c.protoDir),
+		GapicToGenerate:   c.gapicToGenerate,
+		OnlyGenerateGapic: c.onlyGapics,
+		LocalMode:         true,
+	}
+	if _, err := generator.Generate(ctx, conf); err != nil {
+		log.Printf("Generator ran (and failed) in %s\n", tmpDir)
+		log.Fatal(err)
+	}
+	return nil
+}
+
+// gitShallowClone clones a repository into the provided tmpDir if a dir has not
+// been specified.
+func gitShallowClone(eg *errgroup.Group, repo, dir, tmpDir string) {
+	if dir != "" {
+		return
+	}
+	eg.Go(func() error {
+		log.Printf("cloning %s\n", repo)
+
+		_, err := git.PlainClone(tmpDir, false, &git.CloneOptions{
+			URL:      repo,
+			Progress: os.Stdout,
+			Depth:    1,
+			Tags:     git.NoTags,
+		})
+		return err
+	})
+}
+
+// deafultDir returns the default option if dir is not set.
+func deafultDir(def, dir string) string {
+	if dir == "" {
+		return def
+	}
+	return dir
+}