Use on-demand cluster discovery in on-demand extension (#20065)

This adds an odcds_config field to the extension's config, and also allows the extension to be configured per-route. As it stands, it currently works only with routes using cluster-header config.

Risk Level: Medium, extending one extension in an opt-in way.
Testing: Added unit tests and integration tests.
Fixes #2500

Signed-off-by: Krzesimir Nowak <[email protected]>

api/envoy/extensions/filters/http/on_demand/v3/BUILD

@@ -5,5 +5,8 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
 licenses(["notice"])  # Apache 2
 
 api_proto_package(
-    deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
+    deps = [
+        "//envoy/config/core/v3:pkg",
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+    ],
 )

api/envoy/extensions/filters/http/on_demand/v3/on_demand.proto

@@ -2,20 +2,58 @@ syntax = "proto3";
 
 package envoy.extensions.filters.http.on_demand.v3;
 
+import "envoy/config/core/v3/config_source.proto";
+
+import "google/protobuf/duration.proto";
+
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
+import "validate/validate.proto";
 
 option java_package = "io.envoyproxy.envoy.extensions.filters.http.on_demand.v3";
 option java_outer_classname = "OnDemandProto";
 option java_multiple_files = true;
 option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/on_demand/v3;on_demandv3";
 option (udpa.annotations.file_status).package_version_status = ACTIVE;
 
-// [#protodoc-title: OnDemand]
-// IP tagging :ref:`configuration overview <config_http_filters_on_demand>`.
+// [#protodoc-title: On Demand Discovery]
+// On Demand Discovery :ref:`configuration overview <config_http_filters_on_demand>`.
 // [#extension: envoy.filters.http.on_demand]
 
+// Configuration of on-demand CDS.
+message OnDemandCds {
+  // A configuration source for the service that will be used for
+  // on-demand cluster discovery.
+  config.core.v3.ConfigSource source = 1 [(validate.rules).message = {required: true}];
+
+  // xdstp:// resource locator for on-demand cluster collection.
+  string resources_locator = 2;
+
+  // The timeout for on demand cluster lookup. If not set, defaults to 5 seconds.
+  google.protobuf.Duration timeout = 3;
+}
+
+// On Demand Discovery filter config.
 message OnDemand {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.http.on_demand.v2.OnDemand";
+
+  // An optional configuration for on-demand cluster discovery
+  // service. If not specified, the on-demand cluster discovery will
+  // be disabled. When it's specified, the filter will pause the
+  // request to an unknown cluster and will begin a cluster discovery
+  // process. When the discovery is finished (successfully or not), the
+  // request will be resumed for further processing.
+  OnDemandCds odcds = 1;
+}
+
+// Per-route configuration for On Demand Discovery.
+message PerRouteConfig {
+  // An optional configuration for on-demand cluster discovery
+  // service. If not specified, the on-demand cluster discovery will
+  // be disabled. When it's specified, the filter will pause the
+  // request to an unknown cluster and will begin a cluster discovery
+  // process. When the discovery is finished (successfully or not), the
+  // request will be resumed for further processing.
+  OnDemandCds odcds = 1;
 }

docs/root/configuration/http/http_filters/on_demand_updates_filter.rst

@@ -1,9 +1,10 @@
 .. _config_http_filters_on_demand:
 
-On-demand VHDS and S/RDS Updates
-================================
+On-demand VHDS, S/RDS and CDS Updates
+=====================================
 
-The on demand filter can be used to support either on demand VHDS or S/RDS update if configured in the filter chain.
+The on demand filter can be used to support either on demand VHDS or S/RDS update together with on demand CDS update
+if configured in the filter chain.
 
 The on-demand update filter can be used to request a :ref:`virtual host <envoy_v3_api_msg_config.route.v3.VirtualHost>`
 data if it's not already present in the :ref:`Route Configuration <envoy_v3_api_msg_config.route.v3.RouteConfiguration>`. The
@@ -15,10 +16,27 @@ The on-demand update filter can also be used to request a *Route Configuration*
 loaded on demand in the :ref:`Scoped RouteConfiguration <envoy_v3_api_msg_config.route.v3.ScopedRouteConfiguration>`.
 The contents of the HTTP header is used to find the scope and create the on-demand request.
 
+The on-demand update filter can also be used to request a cluster when the target cluster name is only known after receiving
+a request and the cluster is missing - this is a scenario that could happen with
+the :ref:`cluster_header <envoy_v3_api_field_config.route.v3.RouteAction.cluster_header>` route action. For an on demand
+request to be created, :ref:`odcds <envoy_v3_api_field_extensions.filters.http.on_demand.v3.OnDemand.odcds>`
+(or its :ref:`per route counterpart <envoy_v3_api_field_extensions.filters.http.on_demand.v3.PerRouteConfig.odcds>`)
+must be specified and a header specified in :ref:`cluster_header <envoy_v3_api_field_config.route.v3.RouteAction.cluster_header>`
+action must be present in the HTTP request.
+
 On-demand VHDS and on-demand S/RDS can not be used at the same time at this point.
 
+On-demand CDS can also be enabled or disabled per virtual host or route. Specifying an extension config
+in :ref:`virtual host's typed_per_filter_config <envoy_v3_api_field_config.route.v3.VirtualHost.typed_per_filter_config>` or
+the :ref:`route's typed_per_filter_config <envoy_v3_api_field_config.route.v3.Route.typed_per_filter_config>` without
+the :ref:`odcds <envoy_v3_api_field_extensions.filters.http.on_demand.v3.OnDemand.odcds>` field disables
+the on demand CDS for requests using this virtual host or route. Conversely,
+if :ref:`odcds <envoy_v3_api_field_extensions.filters.http.on_demand.v3.OnDemand.odcds>` is specified,
+on demand CDS is enabled for requests using this virtual host or route.
+
 Configuration
 -------------
 * :ref:`v3 API reference <envoy_v3_api_msg_extensions.filters.http.on_demand.v3.OnDemand>`
+* :ref:`v3 API reference for per route/virtual host config <envoy_v3_api_msg_extensions.filters.http.on_demand.v3.PerRouteConfig>`
 * This filter should be configured with the name *envoy.filters.http.on_demand*.
 * The filter should be placed before *envoy.filters.http.router* filter in the HttpConnectionManager's filter chain.

docs/root/version_history/current.rst

@@ -44,6 +44,7 @@ New Features
 * access_log: added new access_log command operators to retrieve upstream connection information: ``%UPSTREAM_PROTOCOL%``, ``%UPSTREAM_PEER_SUBJECT%``, ``%UPSTREAM_PEER_ISSUER%``, ``%UPSTREAM_TLS_SESSION_ID%``, ``%UPSTREAM_TLS_CIPHER%``, ``%UPSTREAM_TLS_VERSION%``, ``%UPSTREAM_PEER_CERT_V_START%``, ``%UPSTREAM_PEER_CERT_V_END%`` and ``%UPSTREAM_PEER_CERT%``.
 * dns_resolver: added :ref:`include_unroutable_families<envoy_v3_api_field_extensions.network.dns_resolver.apple.v3.AppleDnsResolverConfig.include_unroutable_families>` to the Apple DNS resolver.
 * ext_proc: added support for per-route :ref:`grpc_service <envoy_v3_api_field_extensions.filters.http.ext_proc.v3.ExtProcOverrides.grpc_service>`.
+* on_demand: :ref:`OnDemand <envoy_v3_api_msg_extensions.filters.http.on_demand.v3.OnDemand>` got extended to hold configuration for on-demand cluster discovery. A similar message for :ref:`per-route configuration <envoy_v3_api_msg_extensions.filters.http.on_demand.v3.PerRouteConfig>` is also added.
 * thrift: added flag to router to control downstream local close. :ref:`close_downstream_on_upstream_error <envoy_v3_api_field_extensions.filters.network.thrift_proxy.router.v3.Router.close_downstream_on_upstream_error>`.
 
 Deprecated

envoy/common/optref.h

@@ -16,6 +16,7 @@ namespace Envoy {
 template <class T> struct OptRef : public absl::optional<std::reference_wrapper<T>> {
   OptRef(T& t) : absl::optional<std::reference_wrapper<T>>(t) {}
   OptRef() = default;
+  OptRef(absl::nullopt_t) : OptRef() {}
 
   /**
    * Copy constructor that allows conversion.

source/extensions/filters/http/on_demand/BUILD

@@ -19,10 +19,14 @@ envoy_cc_library(
         "//envoy/event:dispatcher_interface",
         "//envoy/http:filter_interface",
         "//envoy/server:filter_config_interface",
+        "//envoy/upstream:cluster_manager_interface",
         "//source/common/common:assert_lib",
         "//source/common/common:enum_to_int",
+        "//source/common/config:xds_resource_lib",
         "//source/common/http:codes_lib",
         "//source/common/http:header_map_lib",
+        "//source/extensions/filters/http:well_known_names",
+        "@envoy_api//envoy/extensions/filters/http/on_demand/v3:pkg_cc_proto",
     ],
 )
 

source/extensions/filters/http/on_demand/config.cc

@@ -10,14 +10,24 @@ namespace HttpFilters {
 namespace OnDemand {
 
 Http::FilterFactoryCb OnDemandFilterFactory::createFilterFactoryFromProtoTyped(
-    const envoy::extensions::filters::http::on_demand::v3::OnDemand&, const std::string&,
-    Server::Configuration::FactoryContext&) {
-  return [](Http::FilterChainFactoryCallbacks& callbacks) -> void {
-    callbacks.addStreamDecoderFilter(
-        std::make_shared<Extensions::HttpFilters::OnDemand::OnDemandRouteUpdate>());
+    const envoy::extensions::filters::http::on_demand::v3::OnDemand& proto_config,
+    const std::string&, Server::Configuration::FactoryContext& context) {
+  OnDemandFilterConfigSharedPtr config = std::make_shared<OnDemandFilterConfig>(
+      proto_config, context.clusterManager(), context.messageValidationVisitor());
+  return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void {
+    callbacks.addStreamDecoderFilter(std::make_shared<OnDemandRouteUpdate>(config));
   };
 }
 
+Router::RouteSpecificFilterConfigConstSharedPtr
+OnDemandFilterFactory::createRouteSpecificFilterConfigTyped(
+    const envoy::extensions::filters::http::on_demand::v3::PerRouteConfig& proto_config,
+    Server::Configuration::ServerFactoryContext& context,
+    ProtobufMessage::ValidationVisitor& validation_visitor) {
+  return std::make_shared<const OnDemandFilterConfig>(proto_config, context.clusterManager(),
+                                                      validation_visitor);
+}
+
 /**
  * Static registration for the on-demand filter. @see RegisterFactory.
  */

source/extensions/filters/http/on_demand/config.h

@@ -4,6 +4,7 @@
 #include "envoy/extensions/filters/http/on_demand/v3/on_demand.pb.validate.h"
 
 #include "source/extensions/filters/http/common/factory_base.h"
+#include "source/extensions/filters/http/well_known_names.h"
 
 namespace Envoy {
 namespace Extensions {
@@ -14,14 +15,20 @@ namespace OnDemand {
  * Config registration for the OnDemand filter. @see NamedHttpFilterConfigFactory.
  */
 class OnDemandFilterFactory
-    : public Common::FactoryBase<envoy::extensions::filters::http::on_demand::v3::OnDemand> {
+    : public Common::FactoryBase<envoy::extensions::filters::http::on_demand::v3::OnDemand,
+                                 envoy::extensions::filters::http::on_demand::v3::PerRouteConfig> {
 public:
-  OnDemandFilterFactory() : FactoryBase("envoy.filters.http.on_demand") {}
+  OnDemandFilterFactory() : FactoryBase(HttpFilterNames::get().OnDemand) {}
 
 private:
   Http::FilterFactoryCb createFilterFactoryFromProtoTyped(
       const envoy::extensions::filters::http::on_demand::v3::OnDemand& proto_config,
       const std::string&, Server::Configuration::FactoryContext& context) override;
+
+  Router::RouteSpecificFilterConfigConstSharedPtr createRouteSpecificFilterConfigTyped(
+      const envoy::extensions::filters::http::on_demand::v3::PerRouteConfig& config,
+      Server::Configuration::ServerFactoryContext& context,
+      ProtobufMessage::ValidationVisitor& visitor) override;
 };
 
 } // namespace OnDemand