Merge pull request cockroachdb#4 from ahrtr/enhance_workflow_20221129

Add script to generate and verify proto files

.github/workflows/test.yaml

@@ -1,4 +1,4 @@
-name: Static Analysis
+name: Test
 on: [push, pull_request]
 jobs:
   run:

Makefile

@@ -1,6 +1,6 @@
 
 .PHONY: verify
-verify: verify-gofmt verify-dep verify-lint verify-mod-tidy
+verify: verify-gofmt verify-dep verify-lint verify-mod-tidy verify-genproto
 
 .PHONY: verify-gofmt
 verify-gofmt:
@@ -18,6 +18,9 @@ verify-lint:
 verify-mod-tidy:
 	PASSES="mod_tidy" ./scripts/test.sh
 
+.PHONY: verify-genproto
+verify-genproto:
+	PASSES="genproto" ./scripts/test.sh
 
 .PHONY: test
 test:

scripts/genproto.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Generate all etcd protobuf bindings.
+# Run from repository root directory named etcd.
+#
+set -e
+shopt -s globstar
+
+if ! [[ "$0" =~ scripts/genproto.sh ]]; then
+  echo "must be run from repository root"
+  exit 255
+fi
+
+source ./scripts/test_lib.sh
+
+if [[ $(protoc --version | cut -f2 -d' ') != "3.14.0" ]]; then
+  echo "could not find protoc 3.14.0, is it installed + in PATH?"
+  exit 255
+fi
+
+GOFAST_BIN=$(tool_get_bin github.com/gogo/protobuf/protoc-gen-gofast)
+GRPC_GATEWAY_BIN=$(tool_get_bin github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway)
+SWAGGER_BIN=$(tool_get_bin github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger)
+GOGOPROTO_ROOT="$(tool_pkg_dir github.com/gogo/protobuf/proto)/.."
+GRPC_GATEWAY_ROOT="$(tool_pkg_dir github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway)/.."
+
+echo
+echo "Resolved binary and packages versions:"
+echo "  - protoc-gen-gofast:       ${GOFAST_BIN}"
+echo "  - protoc-gen-grpc-gateway: ${GRPC_GATEWAY_BIN}"
+echo "  - swagger:                 ${SWAGGER_BIN}"
+echo "  - gogoproto-root:          ${GOGOPROTO_ROOT}"
+echo "  - grpc-gateway-root:       ${GRPC_GATEWAY_ROOT}"
+GOGOPROTO_PATH="${GOGOPROTO_ROOT}:${GOGOPROTO_ROOT}/protobuf"
+
+# directories containing protos to be built
+DIRS="./raftpb"
+
+log_callout -e "\\nRunning gofast (gogo) proto generation..."
+
+for dir in ${DIRS}; do
+  pushd "${dir}"
+    protoc --gofast_out=plugins=grpc:. -I=".:${GOGOPROTO_PATH}:${RAFT_ROOT_DIR}/..:${RAFT_ROOT_DIR}:${GRPC_GATEWAY_ROOT}/third_party/googleapis" \
+      --plugin="${GOFAST_BIN}" ./**/*.proto
+
+    sed -i.bak -E 's|"raft/raftpb"|"go.etcd.io/etcd/raft/v3/raftpb"|g' ./**/*.pb.go
+    sed -i.bak -E 's|"google/protobuf"|"github.com/gogo/protobuf/protoc-gen-gogo/descriptor"|g' ./**/*.pb.go
+
+    rm -f ./**/*.bak
+    gofmt -s -w ./**/*.pb.go
+    run_go_tool "golang.org/x/tools/cmd/goimports" -w ./**/*.pb.go
+  popd
+done
+
+log_success -e "\\n./genproto SUCCESS"

scripts/test.sh

@@ -33,6 +33,10 @@ function gofmt_pass {
   generic_checker go_fmt_for_package
 }
 
+function genproto_pass {
+  "${RAFT_ROOT_DIR}/scripts/verify_genproto.sh"
+}
+
 ######## VARIOUS CHECKERS ######################################################
 
 function dump_deps_of_module() {

scripts/test_lib.sh

@@ -1,5 +1,13 @@
 #!/usr/bin/env bash
 
+ROOT_MODULE="go.etcd.io/raft"
+
+function set_root_dir {
+  RAFT_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3")
+}
+
+set_root_dir
+
 ####   Convenient IO methods #####
 
 COLOR_RED='\033[0;31m'
@@ -48,3 +56,54 @@ function log_info {
   >&2 echo -n -e "${COLOR_NONE}"
 }
 
+# run_for_module [module] [cmd]
+# executes given command in the given module for given pkgs.
+#   module_name - "." (in future: tests, client, server)
+#   cmd         - cmd to be executed - that takes package as last argument
+function run_for_module {
+  local module=${1:-"."}
+  shift 1
+  (
+    cd "${RAFT_ROOT_DIR}/${module}" && "$@"
+  )
+}
+
+# tool_pkg_dir [pkg] - returns absolute path to a directory that stores given pkg.
+# The pkg versions must be defined in ./tools/mod directory.
+function tool_pkg_dir {
+  run_for_module ./tools/mod go list -f '{{.Dir}}' "${1}"
+}
+
+# tool_get_bin [tool] - returns absolute path to a tool binary (or returns error)
+function tool_get_bin {
+  local tool="$1"
+  local pkg_part="$1"
+  if [[ "$tool" == *"@"* ]]; then
+    pkg_part=$(echo "${tool}" | cut -d'@' -f1)
+    # shellcheck disable=SC2086
+    go install ${GOBINARGS:-} "${tool}" || return 2
+  else
+    # shellcheck disable=SC2086
+    run_for_module ./tools/mod go install ${GOBINARGS:-} "${tool}" || return 2
+  fi
+
+  # remove the version suffix, such as removing "/v3" from "go.etcd.io/etcd/v3".
+  local cmd_base_name
+  cmd_base_name=$(basename "${pkg_part}")
+  if [[ ${cmd_base_name} =~ ^v[0-9]*$ ]]; then
+    pkg_part=$(dirname "${pkg_part}")
+  fi
+
+  run_for_module ./tools/mod go list -f '{{.Target}}' "${pkg_part}"
+}
+
+# tool_get_bin [tool]
+function run_go_tool {
+  local cmdbin
+  if ! cmdbin=$(GOARCH="" tool_get_bin "${1}"); then
+    log_warning "Failed to install tool '${1}'"
+    return 2
+  fi
+  shift 1
+  GOARCH="" "${cmdbin}" "$@" || return 2
+}

scripts/verify_genproto.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# This scripts is automatically run by CI to prevent pull requests missing running genproto.sh
+# after changing *.proto file.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+tmpWorkDir=$(mktemp -d -t 'twd.XXXXXX')
+mkdir "$tmpWorkDir/raft"
+tmpWorkDir="$tmpWorkDir/raft"
+cp -r . "$tmpWorkDir"
+pushd "$tmpWorkDir"
+git add -A
+git commit -m init || true # maybe fail because nothing to commit 
+./scripts/genproto.sh
+diff=$(git diff --numstat | awk '{print $3}')
+popd
+if [ -z "$diff" ]; then
+  echo "PASSED genproto-verification!"
+  exit 0
+fi
+echo "Failed genproto-verification!" >&2
+printf "* Found changed files:\n%s\n" "$diff" >&2
+echo "* Please rerun genproto.sh after changing *.proto file" >&2
+echo "* Run ./scripts/genproto.sh" >&2
+exit 1

tools/mod/go.mod

@@ -0,0 +1,88 @@
+module go.etcd.io/raft/tools/v3
+
+go 1.19
+
+require (
+	github.com/alexkohler/nakedret v1.0.0
+	github.com/chzchzchz/goword v0.0.0-20170907005317-a9744cb52b03
+	github.com/coreos/license-bill-of-materials v0.0.0-20190913234955-13baff47494e
+	github.com/gogo/protobuf v1.3.2
+	github.com/google/addlicense v1.0.0
+	github.com/gordonklaus/ineffassign v0.0.0-20210914165742-4cc7213b9bc8
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0
+	github.com/gyuho/gocovmerge v0.0.0-20171205171859-50c7e6afd535
+	github.com/hexfusion/schwag v0.0.0-20211117114134-3ceb0191ccbf
+	github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f
+	github.com/mgechev/revive v1.2.1
+	github.com/mikefarah/yq/v4 v4.24.2
+	go.etcd.io/gofail v0.0.0-20221125214112-fc21f61ba88a
+	go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116
+	gotest.tools/gotestsum v1.7.0
+	gotest.tools/v3 v3.1.0
+	honnef.co/go/tools v0.3.0
+	mvdan.cc/unparam v0.0.0-20220316160445-06cc5682983b
+)
+
+require (
+	github.com/BurntSushi/toml v1.1.0 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/a8m/envsubst v1.3.0 // indirect
+	github.com/akhenakh/hunspellgo v0.0.0-20160221122622-9db38fa26e19 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
+	github.com/bmatcuk/doublestar/v4 v4.0.2 // indirect
+	github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4 // indirect
+	github.com/dnephin/pflag v1.0.7 // indirect
+	github.com/elliotchance/orderedmap v1.4.0 // indirect
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-openapi/analysis v0.21.2 // indirect
+	github.com/go-openapi/errors v0.19.9 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.6 // indirect
+	github.com/go-openapi/loads v0.21.1 // indirect
+	github.com/go-openapi/spec v0.20.4 // indirect
+	github.com/go-openapi/strfmt v0.21.0 // indirect
+	github.com/go-openapi/swag v0.19.15 // indirect
+	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/goccy/go-yaml v1.9.5 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/protobuf v1.3.3 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jinzhu/copier v0.3.5 // indirect
+	github.com/jonboulle/clockwork v0.2.2 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/spf13/cobra v1.4.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/timtadh/data-structures v0.5.3 // indirect
+	github.com/timtadh/lexmachine v0.2.2 // indirect
+	github.com/trustmaster/go-aspell v0.0.0-20200701131845-c2b1f55bec8f // indirect
+	go.mongodb.org/mongo-driver v1.7.3 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86 // indirect
+	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.11-0.20220316014157-77aa08bb151a // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884 // indirect
+	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)