api: add go proto generation script

.circleci/config.yml

@@ -64,6 +64,7 @@ jobs:
            fingerprints:
              - "fb:f3:fe:be:1c:b2:ec:b6:25:f9:7b:a6:87:54:02:8c"
        - run: ci/api_mirror.sh
+       - run: ci/go_mirror.sh
        - store_artifacts:
            path: /build/envoy/generated
            destination: /

api/bazel/api_build_system.bzl

@@ -7,7 +7,7 @@ _PY_SUFFIX = "_py"
 _CC_SUFFIX = "_cc"
 _CC_EXPORT_SUFFIX = "_export_cc"
 _GO_PROTO_SUFFIX = "_go_proto"
-_GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/data-plane-api/api/"
+_GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/go-control-plane/"
 
 _COMMON_PROTO_DEPS = [
     "@com_google_protobuf//:any_proto",

api/bazel/repositories.bzl

@@ -54,7 +54,7 @@ api_proto_library(
 
 go_proto_library(
     name = "client_model_go_proto",
-    importpath = "client_model",
+    importpath = "github.com/prometheus/client_model/go",
     proto = ":client_model",
     visibility = ["//visibility:public"],
 )

api/test/build/go_build_test.go

@@ -3,14 +3,14 @@ package go_build_test
 import (
 	"testing"
 
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/api/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/api/v2/auth"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/config/bootstrap/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/service/accesslog/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/service/discovery/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/service/metrics/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/service/ratelimit/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/service/trace/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/api/v2/auth"
+	_ "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/service/metrics/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/service/ratelimit/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/service/trace/v2"
 )
 
 func TestNoop(t *testing.T) {

ci/go_mirror.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
+then
+  tools/api/generate_go_protobuf.py
+fi

tools/api/generate_go_protobuf.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+from subprocess import check_output
+from subprocess import check_call
+
+import glob
+import os
+import shutil
+import sys
+import re
+
+targets = '@envoy_api//...'
+import_base = 'github.com/envoyproxy/go-control-plane'
+output_base = 'build_go'
+repo_base = 'go-control-plane'
+branch = 'master'
+mirror_msg = 'Mirrored from envoyproxy/envoy @ '
+user_name = 'Kuat Yessenov'
+user_email = '[email protected]'
+
+
+def generateProtobufs(output):
+  bazel_bin = check_output(['bazel', 'info', 'bazel-bin']).decode().strip()
+  go_protos = check_output([
+      'bazel',
+      'query',
+      'kind("go_proto_library", %s)' % targets,
+  ]).split()
+
+  # Each rule has the form @envoy_api//foo/bar:baz_go_proto.
+  # First build all the rules to ensure we have the output files.
+  check_call(['bazel', 'build', '-c', 'fastbuild'] + go_protos)
+
+  shutil.rmtree(output, ignore_errors=True)
+  for rule in go_protos:
+    # Example rule:
+    # @envoy_api//envoy/config/bootstrap/v2:pkg_go_proto
+    #
+    # Example generated directory:
+    # bazel-bin/external/envoy_api/envoy/config/bootstrap/v2/linux_amd64_stripped/pkg_go_proto%/github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2/
+    #
+    # Example output directory:
+    # go_out/envoy/config/bootstrap/v2
+    rule_dir, proto = rule.decode()[len('@envoy_api//'):].rsplit(':', 1)
+
+    input_dir = os.path.join(bazel_bin, 'external', 'envoy_api', rule_dir, 'linux_amd64_stripped',
+                             proto + '%', import_base, rule_dir)
+    input_files = glob.glob(os.path.join(input_dir, '*.go'))
+    output_dir = os.path.join(output, rule_dir)
+
+    # Ensure the output directory exists
+    os.makedirs(output_dir, 0o755, exist_ok=True)
+    for generated_file in input_files:
+      shutil.copy(generated_file, output_dir)
+  print('Go artifacts placed into: ' + output)
+
+
+def cloneGoProtobufs(repo):
+  # Create a local clone of go-control-plane
+  shutil.rmtree(repo, ignore_errors=True)
+  check_call(['git', 'clone', 'git@git:envoyproxy/go-control-plane', repo])
+  check_call(['git', '-C', repo, 'fetch'])
+  check_call(['git', '-C', repo, 'checkout', '-B', branch, 'origin/master'])
+
+
+def findLastSyncSHA(repo):
+  # Determine last envoyproxy/envoy SHA in envoyproxy/go-control-plane
+  last_commit = check_output(
+      ['git', '-C', repo, 'log', '--grep=' + mirror_msg, '-n', '1',
+       '--format=%B']).decode().strip()
+  if last_commit == "":
+    return 'e7f0b7176efdc65f96eb1697b829d1e6187f4502'
+  m = re.search(mirror_msg + '(\w+)', last_commit)
+  return m.group(1)
+
+
+def updatedSinceSHA(repo, last_sha):
+  # Determine if there are changes to api since last SHA
+  changes = check_output(['git', 'rev-list', '%s..HEAD' % last_sha, 'api/envoy']).decode().split()
+  return changes
+
+
+def syncGoProtobufs(output, repo):
+  # Sync generated content against repo and return true if there is a commit necessary
+  dst = os.path.join(repo, 'envoy')
+  # Remove subtree at envoy in repo
+  shutil.rmtree(dst, ignore_errors=True)
+  # Copy subtree at envoy from output to repo
+  shutil.copytree(os.path.join(output, 'envoy'), dst)
+
+
+def publishGoProtobufs(repo, sha):
+  # Publish generated files with the last SHA changes to api
+  check_call(['git', '-C', repo, 'config', 'user.name', user_name])
+  check_call(['git', '-C', repo, 'config', 'user.email', user_email])
+  check_call(['git', '-C', repo, 'add', 'envoy'])
+  check_call(['git', '-C', repo, 'commit', '-s', '-m', mirror_msg + sha])
+  check_call(['git', '-C', repo, 'push', 'origin', branch])
+
+
+if __name__ == "__main__":
+  workspace = check_output(['bazel', 'info', 'workspace']).decode().strip()
+  output = os.path.join(workspace, output_base)
+  generateProtobufs(output)
+  repo = os.path.join(workspace, repo_base)
+  cloneGoProtobufs(repo)
+  last_sha = findLastSyncSHA(repo)
+  changes = updatedSinceSHA(repo, last_sha)
+  if changes:
+    print('Changes detected: %s' % changes)
+    syncGoProtobufs(output, repo)
+    publishGoProtobufs(repo, changes[0])