config: support extrinsic yaml anchor declarations (envoyproxy#16543)

Signed-off-by: Mike Schore <[email protected]>

configs/example_configs_validation.py

@@ -1,22 +1,18 @@
 import pathlib
 import sys
 
-import yaml
-
 from google.protobuf.json_format import ParseError
 
 sys.path = [p for p in sys.path if not p.endswith('bazel_tools')]
 
-from tools.config_validation.validate_fragment import validate_fragment
+from tools.config_validation.validate_fragment import validate_yaml
 
 
 def main():
     errors = []
     for arg in sys.argv[1:]:
         try:
-            validate_fragment(
-                "envoy.config.bootstrap.v3.Bootstrap",
-                yaml.safe_load(pathlib.Path(arg).read_text()))
+            validate_yaml("envoy.config.bootstrap.v3.Bootstrap", pathlib.Path(arg).read_text())
         except (ParseError, KeyError) as e:
             errors.append(arg)
             print(f"\nERROR (validation failed): {arg}\n{e}\n\n")

docs/root/configuration/overview/_include/tagged.yaml

@@ -0,0 +1,43 @@
+!ignore dynamic_sockets:
+  - &admin_address { address: 127.0.0.1, port_value: 9901 }
+  - &listener_address { address: 127.0.0.1, port_value: 10000 }
+  - &lb_address { address: 127.0.0.1, port_value: 1234 }
+
+admin:
+  address:
+    socket_address: *admin_address
+
+static_resources:
+  listeners:
+  - name: listener_0
+    address:
+      socket_address: *listener_address
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress_http
+          codec_type: AUTO
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_service
+              domains: ["*"]
+              routes:
+              - match: { prefix: "/" }
+                route: { cluster: some_service }
+          http_filters:
+          - name: envoy.filters.http.router
+  clusters:
+  - name: some_service
+    connect_timeout: 0.25s
+    type: STATIC
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: some_service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address: *lb_address

docs/root/configuration/overview/examples.rst

@@ -302,3 +302,36 @@ The management server could respond to EDS requests with:
             socket_address:
               address: 127.0.0.2
               port_value: 1234
+
+Special YAML usage
+~~~~~~~~~~~~~~~~~~
+
+When loading YAML configuration, the Envoy loader will interpret map keys tagged with !ignore
+specially, and omit them entirely from the native configuration tree. Ordinarily, the YAML stream
+must adhere strictly to the proto schemas defined for Envoy configuration. This allows content to
+be declared that is explicitly handled as a non-represented type.
+
+This lets you split your file into two parts: one in which we have YAML content not subject to
+parsing according to the schema and another part that is parsed. YAML anchors in the first part
+may be referenced by aliases in the second part. This mechanism can simplify setups that need to
+re-use or dynamically generate configuration fragments.
+
+See the following example:
+
+.. literalinclude:: _include/tagged.yaml
+    :language: yaml
+
+.. warning::
+    If you parse Envoy YAML configuration using external loaders, you may need to inform these
+    loaders about the !ignore tag. Compliant YAML loaders will typically expose an interface to
+    allow you to choose how to handle a custom tag.
+
+For example, this will instruct `PyYAML <https://github.com/yaml/pyyaml>` to treat an ignored
+node as a simple scalar when loading:
+
+.. code-block:: python3
+
+    yaml.SafeLoader.add_constructor('!ignore', yaml.loader.SafeConstructor.construct_scalar)
+
+Alternatively, :repo:`this is how <tools/config_validation/validate_fragment.py>`
+Envoy registers the !ignore tag in config validation.

source/common/protobuf/utility.cc

@@ -108,7 +108,9 @@ ProtobufWkt::Value parseYamlNode(const YAML::Node& node) {
   case YAML::NodeType::Map: {
     auto& struct_fields = *value.mutable_struct_value()->mutable_fields();
     for (const auto& it : node) {
-      struct_fields[it.first.as<std::string>()] = parseYamlNode(it.second);
+      if (it.first.Tag() != "!ignore") {
+        struct_fields[it.first.as<std::string>()] = parseYamlNode(it.second);
+      }
     }
     break;
   }

test/common/protobuf/utility_test.cc

@@ -1185,6 +1185,11 @@ TEST_F(ProtobufUtilityTest, ValueUtilLoadFromYamlObject) {
             "struct_value { fields { key: \"foo\" value { string_value: \"bar\" } } }");
 }
 
+TEST_F(ProtobufUtilityTest, ValueUtilLoadFromYamlObjectWithIgnoredEntries) {
+  EXPECT_EQ(ValueUtil::loadFromYaml("!ignore foo: bar\nbaz: qux").ShortDebugString(),
+            "struct_value { fields { key: \"baz\" value { string_value: \"qux\" } } }");
+}
+
 TEST(LoadFromYamlExceptionTest, BadConversion) {
   std::string bad_yaml = R"EOF(
 admin:

tools/config_validation/validate_fragment.py

@@ -21,6 +21,37 @@
 import argparse
 
 
+class IgnoredKey(yaml.YAMLObject):
+    """Python support type for Envoy's config !ignore tag."""
+    yaml_tag = '!ignore'
+
+    def __init__(self, strval):
+        self.strval = strval
+
+    def __repr__(self):
+        return f'IgnoredKey({str})'
+
+    def __eq__(self, other):
+        return isinstance(other, IgnoredKey) and self.strval == other.strval
+
+    def __hash__(self):
+        return hash((self.yaml_tag, self.strval))
+
+    @classmethod
+    def from_yaml(cls, loader, node):
+        return IgnoredKey(node.value)
+
+    @classmethod
+    def to_yaml(cls, dumper, data):
+        return dumper.represent_scalar(cls.yaml_tag, data.strval)
+
+
+def validate_yaml(type_name, content):
+    yaml.SafeLoader.add_constructor('!ignore', IgnoredKey.from_yaml)
+    yaml.SafeDumper.add_multi_representer(IgnoredKey, IgnoredKey.to_yaml)
+    validate_fragment(type_name, yaml.safe_load(content))
+
+
 def validate_fragment(type_name, fragment):
     """Validate a dictionary representing a JSON/YAML fragment against an Envoy API proto3 type.
 
@@ -33,7 +64,7 @@ def validate_fragment(type_name, fragment):
         fragment: a dictionary representing the parsed JSON/YAML configuration
           fragment.
     """
-    json_fragment = json.dumps(fragment)
+    json_fragment = json.dumps(fragment, skipkeys=True)
 
     r = runfiles.Create()
     all_protos_pb_text_path = r.Rlocation(
@@ -69,4 +100,4 @@ def parse_args():
     message_type = parsed_args.message_type
     content = parsed_args.s if (parsed_args.fragment_path is None) else pathlib.Path(
         parsed_args.fragment_path).read_text()
-    validate_fragment(message_type, yaml.safe_load(content))
+    validate_yaml(message_type, content)