diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 9c765d3b5c9f..1c267f939459 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -291,6 +291,8 @@ elif [[ "$CI_TARGET" == "bazel.fuzzit" ]]; then elif [[ "$CI_TARGET" == "fix_format" ]]; then # proto_format.sh needs to build protobuf. setup_clang_toolchain + echo "protoxform_test..." + ./tools/protoxform_test.sh echo "fix_format..." ./tools/check_format.py fix ./tools/format_python_tools.sh fix diff --git a/tools/BUILD b/tools/BUILD index e3f44a5f9646..a4699559aedc 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -25,6 +25,15 @@ envoy_py_test_binary( ], ) +py_library( + name = "run_command", + srcs = [ + "run_command.py", + ], + srcs_version = "PY3", + visibility = ["//visibility:public"], +) + envoy_cc_binary( name = "bootstrap2pb", srcs = ["bootstrap2pb.cc"], diff --git a/tools/check_format_test_helper.py b/tools/check_format_test_helper.py index dbe943dad7ca..0a24476420fc 100755 --- a/tools/check_format_test_helper.py +++ b/tools/check_format_test_helper.py @@ -7,6 +7,7 @@ from __future__ import print_function +from run_command import runCommand import argparse import logging import os @@ -21,30 +22,13 @@ errors = 0 -# Echoes and runs an OS command, returning exit status and the captured -# stdout+stderr as a string array. -def runCommand(command): - stdout = [] - status = 0 - try: - out = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT).strip() - if out: - stdout = out.decode('utf-8').split("\n") - except subprocess.CalledProcessError as e: - status = e.returncode - for line in e.output.splitlines(): - stdout.append(line) - logging.info("%s" % command) - return status, stdout - - # Runs the 'check_format' operation, on the specified file, printing # the comamnd run and the status code as well as the stdout, and returning # all of that to the caller. def runCheckFormat(operation, filename): command = check_format + " " + operation + " " + filename - status, stdout = runCommand(command) - return (command, status, stdout) + status, stdout, stderr = runCommand(command) + return (command, status, stdout + stderr) def getInputFile(filename, extra_input_files=None): @@ -80,10 +64,10 @@ def fixFileExpectingSuccess(file, extra_input_files=None): print("FAILED:") emitStdoutAsError(stdout) return 1 - status, stdout = runCommand('diff ' + outfile + ' ' + infile + '.gold') + status, stdout, stderr = runCommand('diff ' + outfile + ' ' + infile + '.gold') if status != 0: print("FAILED:") - emitStdoutAsError(stdout) + emitStdoutAsError(stdout + stderr) return 1 return 0 @@ -92,7 +76,7 @@ def fixFileExpectingNoChange(file): command, infile, outfile, status, stdout = fixFileHelper(file) if status != 0: return 1 - status, stdout = runCommand('diff ' + outfile + ' ' + infile) + status, stdout, stderr = runCommand('diff ' + outfile + ' ' + infile) if status != 0: logging.error(file + ': expected file to remain unchanged') return 1 @@ -100,7 +84,7 @@ def fixFileExpectingNoChange(file): def emitStdoutAsError(stdout): - logging.error("\n".join(line.decode('utf-8') for line in stdout)) + logging.error("\n".join(stdout)) def expectError(filename, status, stdout, expected_substring): @@ -108,7 +92,7 @@ def expectError(filename, status, stdout, expected_substring): logging.error("%s: Expected failure `%s`, but succeeded" % (filename, expected_substring)) return 1 for line in stdout: - if expected_substring in line.decode('utf-8'): + if expected_substring in line: return 0 logging.error("%s: Could not find '%s' in:\n" % (filename, expected_substring)) emitStdoutAsError(stdout) @@ -283,4 +267,4 @@ def runChecks(): if errors != 0: logging.error("%d FAILURES" % errors) exit(1) - logging.warning("PASS") + logging.warning("PASS") \ No newline at end of file diff --git a/tools/protoxform/protoxform.py b/tools/protoxform/protoxform.py index 7ebb170a2480..c88f7b64ba0e 100755 --- a/tools/protoxform/protoxform.py +++ b/tools/protoxform/protoxform.py @@ -535,4 +535,4 @@ def Main(): if __name__ == '__main__': - Main() + Main() \ No newline at end of file diff --git a/tools/protoxform_test.sh b/tools/protoxform_test.sh new file mode 100755 index 000000000000..705a8235e1c0 --- /dev/null +++ b/tools/protoxform_test.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +rm -rf bazel-bin/tools + +declare -r PROTO_TARGETS=$(bazel query "labels(srcs, labels(deps, //tools/testdata/protoxform:protos))") + +BAZEL_BUILD_OPTIONS+=" --remote_download_outputs=all" + +bazel build ${BAZEL_BUILD_OPTIONS} --//tools/api_proto_plugin:default_type_db_target=//tools/testdata/protoxform:protos \ + //tools/testdata/protoxform:protos --aspects //tools/protoxform:protoxform.bzl%protoxform_aspect --output_groups=proto \ + --action_env=CPROFILE_ENABLED=1 --host_force_python=PY3 + +./tools/protoxform_test_helper.py ${PROTO_TARGETS} \ No newline at end of file diff --git a/tools/protoxform_test_helper.py b/tools/protoxform_test_helper.py new file mode 100755 index 000000000000..51a26d92d031 --- /dev/null +++ b/tools/protoxform_test_helper.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 + +from run_command import runCommand + +import logging +import os +import re +import sys + + +def PathAndFilename(label): + """Retrieve actual path and filename from bazel label + + Args: + label: bazel label to specify target proto. + + Returns: + actual path and filename + """ + if label.startswith('/'): + label = label.replace('//', '/', 1) + elif label.startswith('@'): + label = re.sub(r'@.*/', '/', label) + else: + return label + label = label.replace(":", "/") + splitted_label = label.split('/') + return ['/'.join(splitted_label[:len(splitted_label) - 1]), splitted_label[-1]] + + +def GoldenProtoFile(path, filename, version): + """Retrieve golden proto file path. In general, those are placed in tools/testdata/protoxform. + + Args: + path: target proto path + filename: target proto filename + version: api version to specify target golden proto filename + + Returns: + actual golden proto absolute path + """ + base = "./" + base += path + "/" + filename + "." + version + ".gold" + return os.path.abspath(base) + + +def ResultProtoFile(path, filename, version): + """Retrieve result proto file path. In general, those are placed in bazel artifacts. + + Args: + path: target proto path + filename: target proto filename + version: api version to specify target result proto filename + + Returns: + actual result proto absolute path + """ + base = "./bazel-bin" + base += os.path.join(path, "protos") + base += os.path.join(base, path) + base += "/{0}.{1}.proto".format(filename, version) + return os.path.abspath(base) + + +def Diff(result_file, golden_file): + """Execute diff command with unified form + + Args: + result_file: result proto file + golden_file: golden proto file + + Returns: + output and status code + """ + command = 'diff -u ' + command += result_file + ' ' + command += golden_file + status, stdout, stderr = runCommand(command) + return [status, stdout, stderr] + + +def Run(path, filename, version): + """Run main execution for protoxform test + + Args: + path: target proto path + filename: target proto filename + version: api version to specify target result proto filename + + Returns: + result message extracted from diff command + """ + message = "" + golden_path = GoldenProtoFile(path, filename, version) + test_path = ResultProtoFile(path, filename, version) + + status, stdout, stderr = Diff(test_path, golden_path) + + if status != 0: + message = '\n'.join([str(line) for line in stdout + stderr]) + + return message + + +if __name__ == "__main__": + messages = "" + logging.basicConfig(format='%(message)s') + path, filename = PathAndFilename(sys.argv[1]) + messages += Run(path, filename, 'v2') + messages += Run(path, filename, 'v3alpha') + + if len(messages) == 0: + logging.warning("PASS") + sys.exit(0) + else: + logging.error("FAILED:\n{}".format(messages)) + sys.exit(1) diff --git a/tools/run_command.py b/tools/run_command.py new file mode 100644 index 000000000000..0b75bedfc141 --- /dev/null +++ b/tools/run_command.py @@ -0,0 +1,10 @@ +import subprocess + + +# Echoes and runs an OS command, returning exit status and the captured +# stdout and stderr as a string array. +def runCommand(command): + proc = subprocess.run([command], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + return proc.returncode, proc.stdout.decode('utf-8').split('\n'), proc.stderr.decode( + 'utf-8').split('\n') diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index a0d44a8319c5..ed6c2f16adbe 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -13,6 +13,7 @@ API ASAN ASCII ASSERTs +AST AWS BACKTRACE BSON @@ -302,6 +303,7 @@ WKT WRR WS Welford's +Wi XDS XFCC XFF @@ -419,6 +421,7 @@ codings combinatorial comparator compat +compiation completable cond condvar @@ -525,6 +528,7 @@ evwatch exe execlp expectable +extrahelp faceplant facto failover @@ -645,6 +649,7 @@ lexically libc libevent libstdc +libtooling lifecycle lightstep linearization @@ -652,6 +657,7 @@ linearize linearized linux livelock +llvm localhost lockless login @@ -665,6 +671,7 @@ lowp ltrim lua lyft +mabye maglev malloc matchable diff --git a/tools/testdata/protoxform/BUILD b/tools/testdata/protoxform/BUILD new file mode 100644 index 000000000000..2ffec2e74aef --- /dev/null +++ b/tools/testdata/protoxform/BUILD @@ -0,0 +1,9 @@ +licenses(["notice"]) # Apache 2 + +proto_library( + name = "protos", + visibility = ["//visibility:public"], + deps = [ + "//tools/testdata/protoxform/envoy/v2:protos", + ], +) diff --git a/tools/testdata/protoxform/envoy/v2/BUILD b/tools/testdata/protoxform/envoy/v2/BUILD new file mode 100644 index 000000000000..dc55a2a63638 --- /dev/null +++ b/tools/testdata/protoxform/envoy/v2/BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # Apache 2 + +proto_library( + name = "protos", + srcs = [ + "sample.proto", + ], + visibility = ["//visibility:public"], + deps = [ + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/tools/testdata/protoxform/envoy/v2/sample.proto b/tools/testdata/protoxform/envoy/v2/sample.proto new file mode 100644 index 000000000000..79f91b301505 --- /dev/null +++ b/tools/testdata/protoxform/envoy/v2/sample.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package envoy.v2; + +import "udpa/annotations/migrate.proto"; + +message Sample { + message Entry { + string key = 1; + string value = 2; + } + repeated Entry entries = 1; + string will_deprecated = 2 [deprecated = true]; + string will_rename_compoent = 3 [(udpa.annotations.field_migrate).rename = "renamed_component"]; +} diff --git a/tools/testdata/protoxform/envoy/v2/sample.proto.v2.gold b/tools/testdata/protoxform/envoy/v2/sample.proto.v2.gold new file mode 100644 index 000000000000..b47a8bccb048 --- /dev/null +++ b/tools/testdata/protoxform/envoy/v2/sample.proto.v2.gold @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package envoy.v2; + +option java_package = "io.envoyproxy.envoy.v2"; +option java_outer_classname = "SampleProto"; +option java_multiple_files = true; + +import "udpa/annotations/migrate.proto"; + +message Sample { + message Entry { + string key = 1; + + string value = 2; + } + + repeated Entry entries = 1; + + string will_deprecated = 2 [deprecated = true]; + + string will_rename_compoent = 3 [(udpa.annotations.field_migrate).rename = "renamed_component"]; +} diff --git a/tools/testdata/protoxform/envoy/v2/sample.proto.v3alpha.gold b/tools/testdata/protoxform/envoy/v2/sample.proto.v3alpha.gold new file mode 100644 index 000000000000..f88430db2256 --- /dev/null +++ b/tools/testdata/protoxform/envoy/v2/sample.proto.v3alpha.gold @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package envoy.v3alpha; + +option java_package = "io.envoyproxy.envoy.v3alpha"; +option java_outer_classname = "SampleProto"; +option java_multiple_files = true; + +import "udpa/annotations/versioning.proto"; + +message Sample { + option (udpa.annotations.versioning).previous_message_type = "envoy.v2.Sample"; + + message Entry { + option (udpa.annotations.versioning).previous_message_type = "envoy.v2.Sample.Entry"; + + string key = 1; + + string value = 2; + } + + reserved 2; + + reserved "will_deprecated"; + + repeated Entry entries = 1; + + string renamed_component = 3; +} diff --git a/tools/type_whisperer/type_database.bzl b/tools/type_whisperer/type_database.bzl index 6311d27afc8e..bc0e63a1c242 100644 --- a/tools/type_whisperer/type_database.bzl +++ b/tools/type_whisperer/type_database.bzl @@ -4,12 +4,12 @@ def _type_database_impl(ctx): type_db_deps = [] for target in ctx.attr.targets: type_db_deps.append(target[OutputGroupInfo].types_pb_text) - type_db_deps = depset(transitive = type_db_deps) args = [ctx.outputs.pb_text.path] for dep in type_db_deps.to_list(): - if dep.owner.workspace_name in ctx.attr.proto_repositories: + ws_name = dep.owner.workspace_name + if (not ws_name) or ws_name in ctx.attr.proto_repositories: args.append(dep.path) ctx.actions.run( diff --git a/tools/type_whisperer/typedb_gen.py b/tools/type_whisperer/typedb_gen.py index 317912a2cae1..e6fbb0d863be 100644 --- a/tools/type_whisperer/typedb_gen.py +++ b/tools/type_whisperer/typedb_gen.py @@ -22,8 +22,8 @@ # As with TYPE_UPGRADE_REGEXES but for API .proto paths. PATH_UPGRADE_REGEXES = [ - (r'(envoy/[\w/]*/)(v1alpha\d?|v1)', r'\1v3alpha'), - (r'(envoy/[\w/]*/)(v2alpha\d?|v2)', r'\1v3alpha'), + (r'(envoy/([\w/]*/|))(v1alpha\d?|v1)', r'\1v3alpha'), + (r'(envoy/([\w/]*/|))(v2alpha\d?|v2)', r'\1v3alpha'), # These are special cases, e.g. upgrading versionless packages. ('envoy/type/matcher', 'envoy/type/matcher/v3alpha'), ('envoy/type', 'envoy/type/v3alpha'), @@ -34,6 +34,7 @@ # structural change to have the APIs follow the x/y//vN/z.proto structure of # organization. PKG_FORCE_UPGRADE = [ + 'envoy.v2', 'envoy.api.v2', 'envoy.api.v2.auth', 'envoy.api.v2.cluster',