tools: Add protoxform tests

tools/BUILD

@@ -25,6 +25,15 @@ envoy_py_test_binary(
     ],
 )
 
+py_library(
+    name = "run_command",
+    srcs = [
+        "run_command.py",
+    ],
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+)
+
 envoy_cc_binary(
     name = "bootstrap2pb",
     srcs = ["bootstrap2pb.cc"],

tools/api_proto_plugin/utils.py

@@ -2,7 +2,7 @@
 import os
 
 
-def ProtoFileCanonicalFromLabel(label):
+def ProtoFileCanonicalFromLabel(label, repo_tag):
   """Compute path from API root to a proto file from a Bazel proto label.
 
   Args:
@@ -12,11 +12,11 @@ def ProtoFileCanonicalFromLabel(label):
     A string with the path, e.g. for @envoy_api//envoy/type/matcher:metadata.proto
     this would be envoy/type/matcher/matcher.proto.
   """
-  assert (label.startswith('@envoy_api//'))
-  return label[len('@envoy_api//'):].replace(':', '/')
+  assert (label.startswith(repo_tag))
+  return label[len(repo_tag):].replace(':', '/')
 
 
-def BazelBinPathForOutputArtifact(label, suffix, root=''):
+def BazelBinPathForOutputArtifact(label, suffix, repo_tag, root=''):
   """Find the location in bazel-bin/ for an api_proto_plugin output file.
 
   Args:
@@ -36,5 +36,17 @@ def BazelBinPathForOutputArtifact(label, suffix, root=''):
   # dependencies in the aspect above, they all look the same. So, just pick an
   # arbitrary match and we're done.
   glob_pattern = os.path.join(
-      root, 'bazel-bin/external/envoy_api/**/%s%s' % (ProtoFileCanonicalFromLabel(label), suffix))
+      root, 'bazel-bin/**/%s%s' % (ProtoFileCanonicalFromLabel(label, repo_tag), suffix))
   return glob.glob(glob_pattern, recursive=True)[0]
+
+
+def ExtractRepoName(label):
+  """Extract repository name from label"""
+  repo = ""
+  for i, ch in enumerate(label):
+    if label[i] == label[i + 1]:
+      repo += "//"
+      break
+    else:
+      repo += ch
+  return repo

tools/check_format_test_helper.py

@@ -283,4 +283,4 @@ def runChecks():
   if errors != 0:
     logging.error("%d FAILURES" % errors)
     exit(1)
-  logging.warning("PASS")
+  logging.warning("PASS")

tools/proto_format.sh

@@ -31,4 +31,4 @@ bazel build ${BAZEL_BUILD_OPTIONS} @envoy_api//docs:protos --aspects \
   tools/protoxform/protoxform.bzl%protoxform_aspect --output_groups=proto --action_env=CPROFILE_ENABLED=1 \
   --action_env=TYPE_DB_PATH --host_force_python=PY3
 
-./tools/proto_sync.py "$1" ${PROTO_TARGETS}
+./tools/proto_sync.py "$1" "@envoy_api" ${PROTO_TARGETS}

tools/proto_sync.py

@@ -59,7 +59,7 @@ def __init__(self, message):
         message)
 
 
-def LabelPaths(label, src_suffix):
+def LabelPaths(label, src_suffix, repo_tag):
   """Compute single proto file source/destination paths from a Bazel proto label.
 
   Args:
@@ -72,8 +72,8 @@ def LabelPaths(label, src_suffix):
       destination is a provisional path in the Envoy source tree for copying the
       contents of source when run in fix mode.
   """
-  src = utils.BazelBinPathForOutputArtifact(label, src_suffix)
-  dst = 'api/%s' % utils.ProtoFileCanonicalFromLabel(label)
+  src = utils.BazelBinPathForOutputArtifact(label, src_suffix, repo_tag=repo_tag)
+  dst = 'api/%s' % utils.ProtoFileCanonicalFromLabel(label, repo_tag)
   return src, dst
 
 
@@ -94,27 +94,27 @@ def SyncProtoFile(cmd, src, dst):
       raise RequiresReformatError('%s and %s do not match' % (src, dst))
 
 
-def SyncV2(cmd, src_labels):
+def SyncV2(cmd, src_labels, repo_tag):
   """Diff or in-place update v2 protos from protoxform.py Bazel cache artifacts."
 
   Args:
     cmd: 'check' or 'fix'.
     src_labels: Bazel label for source protos.
   """
   for s in src_labels:
-    src, dst = LabelPaths(s, '.v2.proto')
+    src, dst = LabelPaths(s, '.v2.proto', repo_tag)
     SyncProtoFile(cmd, src, dst)
 
 
-def SyncV3Alpha(cmd, src_labels):
+def SyncV3Alpha(cmd, src_labels, repo_tag):
   """Diff or in-place update v3alpha protos from protoxform.py Bazel cache artifacts."
 
   Args:
     cmd: 'check' or 'fix'.
     src_labels: Bazel label for source protos.
   """
   for s in src_labels:
-    src, dst = LabelPaths(s, '.v3alpha.proto')
+    src, dst = LabelPaths(s, '.v3alpha.proto', repo_tag)
     # Skip empty files, this indicates this file isn't modified in next version.
     if os.stat(src).st_size == 0:
       continue
@@ -257,10 +257,11 @@ def SyncBuildFiles(cmd):
 
 if __name__ == '__main__':
   cmd = sys.argv[1]
-  src_labels = sys.argv[2:]
+  repo_tag = sys.argv[2]
+  src_labels = sys.argv[3:]
   try:
-    SyncV2(cmd, src_labels)
-    SyncV3Alpha(cmd, src_labels)
+    SyncV2(cmd, src_labels, repo_tag)
+    SyncV3Alpha(cmd, src_labels, repo_tag)
     SyncBuildFiles(cmd)
   except ProtoSyncError as e:
     sys.stderr.write('%s\n' % e)

tools/protoxform/options.py

@@ -7,7 +7,7 @@ def AddHideOption(options):
   """Mark message/enum/field/enum value as hidden.
 
   Hidden messages are ignored when generating output.
-
+  
   Args:
     options: MessageOptions/EnumOptions/FieldOptions/EnumValueOptions message.
   """

tools/protoxform/protoxform.py

@@ -57,7 +57,6 @@ def ClangFormat(contents):
 
 def FormatBlock(block):
   """Append \n to a .proto section (e.g.
-
   comment, message definition, etc.) if non-empty.
 
   Args:
@@ -73,7 +72,6 @@ def FormatBlock(block):
 
 def FormatComments(comments):
   """Format a list of comment blocks from SourceCodeInfo.
-
   Prefixes // to each line, separates blocks by spaces.
 
   Args:
@@ -532,4 +530,4 @@ def Main():
 
 
 if __name__ == '__main__':
-  Main()
+  Main()

tools/protoxform_test.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+declare -x PROTO_TARGETS=//tools/testdata/protoxform:protos
+
+BAZEL_BUILD_OPTIONS+=" --remote_download_outputs=all --strategy=protoxform=sandboxed,local"
+bazel build ${BAZEL_BUILD_OPTIONS} ${PROTO_TARGETS} --aspects \
+  tools/type_whisperer/type_whisperer.bzl%type_whisperer_aspect --output_groups=types_pb_text \
+  --host_force_python=PY3
+declare -r PROTO_FILES=$(bazel query "labels(srcs, //tools/testdata/protoxform:protos)")
+declare -x -r TYPE_DB_PATH="${PWD}"/source/common/config/api_type_db.generated.pb_text
+bazel run ${BAZEL_BUILD_OPTIONS} //tools/type_whisperer:typedb_gen -- \
+  ${PWD} ${TYPE_DB_PATH} ${PROTO_FILES}
+bazel build ${BAZEL_BUILD_OPTIONS} ${PROTO_TARGETS} --aspects \
+  tools/protoxform/protoxform.bzl%protoxform_aspect --output_groups=proto --action_env=CPROFILE_ENABLED=1 \
+  --action_env=TYPE_DB_PATH --host_force_python=PY3
+
+./tools/protoxform_test_helper.py ${PROTO_FILES}

tools/protoxform_test_helper.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+from run_command import runCommand
+
+import os
+import re
+import sys
+
+
+def PathAndFilename(label):
+  if label.startswith('/'):
+    label = label.replace('//', '', 1)
+  elif label.startswith('@'):
+    label = re.sub(r'@.*/', '', label)
+  else:
+    return label
+  label = label.replace(":", "/")
+  splitted_label = label.split('/')
+  return ['/'.join(splitted_label[:len(splitted_label) - 1]), splitted_label[-1]]
+
+
+def GoldenProtoFile(path, file, version):
+  path += "/" + file + "." + version + ".gold"
+  return os.path.abspath(path)
+
+
+def ResultProtoFile(path, file, version):
+  base = "./bazel-bin/"
+  base += path + "/protos/" + path + "/" + filename + "." + version + ".proto"
+  return os.path.abspath(base)
+
+
+def diff(result_file, golden_file):
+  command = 'diff '
+  command += result_file + ' '
+  command += golden_file + ' '
+  print(command)
+  status, stdout = runCommand(command)
+  return [status, stdout]
+
+
+if __name__ == "__main__":
+  path, filename = PathAndFilename(sys.argv[1])
+  golden_path_v2 = GoldenProtoFile(path, filename, 'v2')
+  golden_path_v3 = GoldenProtoFile(path, filename, 'v3alpha')
+  test_path_v2 = ResultProtoFile(path, filename, 'v2')
+  test_path_v3 = ResultProtoFile(path, filename, 'v3alpha')
+
+  print(diff(test_path_v2, golden_path_v2))
+  print(diff(test_path_v3, golden_path_v3))

tools/run_command.py

@@ -0,0 +1,19 @@
+import logging
+import subprocess
+
+
+# Echoes and runs an OS command, returning exit status and the captured
+# stdout+stderr as a string array.
+def runCommand(command):
+  stdout = []
+  status = 0
+  try:
+    out = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT).strip()
+    if out:
+      stdout = out.decode('utf-8').split("\n")
+  except subprocess.CalledProcessError as e:
+    status = e.returncode
+    for line in e.output.splitlines():
+      stdout.append(line)
+  logging.info("%s" % command)
+  return status, stdout

tools/testdata/protoxform/BUILD

@@ -0,0 +1,9 @@
+licenses(["notice"])  # Apache 2
+
+proto_library(
+    name = "protos",
+    srcs = [
+        "sample.proto",
+    ],
+    visibility = ["//visibility:public"],
+)

tools/testdata/protoxform/sample.proto

@@ -0,0 +1,5 @@
+syntax = "proto3";
+
+message Sample {
+  string sample = 1;
+}

tools/testdata/protoxform/sample.proto.v2.gold

@@ -0,0 +1,11 @@
+syntax = "proto3";
+
+package;
+
+option java_outer_classname = "SampleProto";
+option java_multiple_files = true;
+option java_package = "io.envoyproxy.";
+
+message Sample {
+  string sample = 1;
+}

tools/type_whisperer/typedb_gen.py

@@ -7,7 +7,7 @@
 
 from google.protobuf import text_format
 
-from tools.api_proto_plugin.utils import BazelBinPathForOutputArtifact
+from tools.api_proto_plugin.utils import BazelBinPathForOutputArtifact, ExtractRepoName
 from tools.type_whisperer.api_type_db_pb2 import TypeDb
 from tools.type_whisperer.types_pb2 import Types, TypeDescription
 
@@ -127,7 +127,10 @@ def NextVersionUpgrade(type_name, type_map, next_version_upgrade_memo, visited=N
 
   # Load type descriptors for each .proto.
   type_desc_paths = [
-      BazelBinPathForOutputArtifact(label, '.types.pb_text', root=src_root) for label in src_labels
+      BazelBinPathForOutputArtifact(label,
+                                    '.types.pb_text',
+                                    root=src_root,
+                                    repo_tag=ExtractRepoName(label)) for label in src_labels
   ]
   type_whispers = map(LoadTypes, type_desc_paths)
   # Aggregate type descriptors to a single type map.