diff --git a/configs/example_configs_validation.py b/configs/example_configs_validation.py index ce8095ee84e2..e05f52b43a44 100644 --- a/configs/example_configs_validation.py +++ b/configs/example_configs_validation.py @@ -1,22 +1,18 @@ import pathlib import sys -import yaml - from google.protobuf.json_format import ParseError sys.path = [p for p in sys.path if not p.endswith('bazel_tools')] -from tools.config_validation.validate_fragment import validate_fragment +from tools.config_validation.validate_fragment import validate_yaml def main(): errors = [] for arg in sys.argv[1:]: try: - validate_fragment( - "envoy.config.bootstrap.v3.Bootstrap", - yaml.safe_load(pathlib.Path(arg).read_text())) + validate_yaml("envoy.config.bootstrap.v3.Bootstrap", pathlib.Path(arg).read_text()) except (ParseError, KeyError) as e: errors.append(arg) print(f"\nERROR (validation failed): {arg}\n{e}\n\n") diff --git a/docs/root/configuration/overview/_include/tagged.yaml b/docs/root/configuration/overview/_include/tagged.yaml new file mode 100644 index 000000000000..a333ec7ffda1 --- /dev/null +++ b/docs/root/configuration/overview/_include/tagged.yaml @@ -0,0 +1,43 @@ +!ignore dynamic_sockets: + - &admin_address { address: 127.0.0.1, port_value: 9901 } + - &listener_address { address: 127.0.0.1, port_value: 10000 } + - &lb_address { address: 127.0.0.1, port_value: 1234 } + +admin: + address: + socket_address: *admin_address + +static_resources: + listeners: + - name: listener_0 + address: + socket_address: *listener_address + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + codec_type: AUTO + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: some_service } + http_filters: + - name: envoy.filters.http.router + clusters: + - name: some_service + connect_timeout: 0.25s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: some_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: *lb_address diff --git a/docs/root/configuration/overview/examples.rst b/docs/root/configuration/overview/examples.rst index f6d14cf5df4b..03811aa1952a 100644 --- a/docs/root/configuration/overview/examples.rst +++ b/docs/root/configuration/overview/examples.rst @@ -302,3 +302,36 @@ The management server could respond to EDS requests with: socket_address: address: 127.0.0.2 port_value: 1234 + +Special YAML usage +~~~~~~~~~~~~~~~~~~ + +When loading YAML configuration, the Envoy loader will interpret map keys tagged with !ignore +specially, and omit them entirely from the native configuration tree. Ordinarily, the YAML stream +must adhere strictly to the proto schemas defined for Envoy configuration. This allows content to +be declared that is explicitly handled as a non-represented type. + +This lets you split your file into two parts: one in which we have YAML content not subject to +parsing according to the schema and another part that is parsed. YAML anchors in the first part +may be referenced by aliases in the second part. This mechanism can simplify setups that need to +re-use or dynamically generate configuration fragments. + +See the following example: + +.. literalinclude:: _include/tagged.yaml + :language: yaml + +.. warning:: + If you parse Envoy YAML configuration using external loaders, you may need to inform these + loaders about the !ignore tag. Compliant YAML loaders will typically expose an interface to + allow you to choose how to handle a custom tag. + +For example, this will instruct `PyYAML ` to treat an ignored +node as a simple scalar when loading: + +.. code-block:: python3 + + yaml.SafeLoader.add_constructor('!ignore', yaml.loader.SafeConstructor.construct_scalar) + +Alternatively, :repo:`this is how ` +Envoy registers the !ignore tag in config validation. diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index f29d8f6125eb..806aca2aec96 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -108,7 +108,9 @@ ProtobufWkt::Value parseYamlNode(const YAML::Node& node) { case YAML::NodeType::Map: { auto& struct_fields = *value.mutable_struct_value()->mutable_fields(); for (const auto& it : node) { - struct_fields[it.first.as()] = parseYamlNode(it.second); + if (it.first.Tag() != "!ignore") { + struct_fields[it.first.as()] = parseYamlNode(it.second); + } } break; } diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 7165b4ec07d7..45983670a003 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -1185,6 +1185,11 @@ TEST_F(ProtobufUtilityTest, ValueUtilLoadFromYamlObject) { "struct_value { fields { key: \"foo\" value { string_value: \"bar\" } } }"); } +TEST_F(ProtobufUtilityTest, ValueUtilLoadFromYamlObjectWithIgnoredEntries) { + EXPECT_EQ(ValueUtil::loadFromYaml("!ignore foo: bar\nbaz: qux").ShortDebugString(), + "struct_value { fields { key: \"baz\" value { string_value: \"qux\" } } }"); +} + TEST(LoadFromYamlExceptionTest, BadConversion) { std::string bad_yaml = R"EOF( admin: diff --git a/tools/config_validation/validate_fragment.py b/tools/config_validation/validate_fragment.py index e708a228a0ce..8059ab3f0c72 100644 --- a/tools/config_validation/validate_fragment.py +++ b/tools/config_validation/validate_fragment.py @@ -21,6 +21,37 @@ import argparse +class IgnoredKey(yaml.YAMLObject): + """Python support type for Envoy's config !ignore tag.""" + yaml_tag = '!ignore' + + def __init__(self, strval): + self.strval = strval + + def __repr__(self): + return f'IgnoredKey({str})' + + def __eq__(self, other): + return isinstance(other, IgnoredKey) and self.strval == other.strval + + def __hash__(self): + return hash((self.yaml_tag, self.strval)) + + @classmethod + def from_yaml(cls, loader, node): + return IgnoredKey(node.value) + + @classmethod + def to_yaml(cls, dumper, data): + return dumper.represent_scalar(cls.yaml_tag, data.strval) + + +def validate_yaml(type_name, content): + yaml.SafeLoader.add_constructor('!ignore', IgnoredKey.from_yaml) + yaml.SafeDumper.add_multi_representer(IgnoredKey, IgnoredKey.to_yaml) + validate_fragment(type_name, yaml.safe_load(content)) + + def validate_fragment(type_name, fragment): """Validate a dictionary representing a JSON/YAML fragment against an Envoy API proto3 type. @@ -33,7 +64,7 @@ def validate_fragment(type_name, fragment): fragment: a dictionary representing the parsed JSON/YAML configuration fragment. """ - json_fragment = json.dumps(fragment) + json_fragment = json.dumps(fragment, skipkeys=True) r = runfiles.Create() all_protos_pb_text_path = r.Rlocation( @@ -69,4 +100,4 @@ def parse_args(): message_type = parsed_args.message_type content = parsed_args.s if (parsed_args.fragment_path is None) else pathlib.Path( parsed_args.fragment_path).read_text() - validate_fragment(message_type, yaml.safe_load(content)) + validate_yaml(message_type, content)