From 38adf1f02e95cf7a7078cdaa39032b62ca1e2ebf Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Sat, 16 Nov 2019 12:42:59 -0800 Subject: [PATCH 01/29] Revert "config: minor cleanup: remove DeltaSubscriptionState::resource_names_ (#8918)" (#9044) This reverts commit 80aedc1c4a1aecc1616bd1563450c69d04e9568f. Revert "config: rename NewGrpcMuxImpl -> GrpcMuxImpl (#8919)" This reverts commit 6d505533304731fcc97041adce1f735431a703d7. Revert "config: reinstate #8478 (unification of delta and SotW xDS), reverted by #8939 (#8974)" This reverts commit a37522cf3f15639c8afeb7402f505044815fcf85. Signed-off-by: Matt Klein --- api/envoy/api/v2/core/config_source.proto | 11 +- .../api/v3alpha/core/config_source.proto | 11 +- include/envoy/config/grpc_mux.h | 89 +++- source/common/config/BUILD | 95 ++-- source/common/config/README.md | 48 +- ...ion_impl.cc => delta_subscription_impl.cc} | 52 +-- .../common/config/delta_subscription_impl.h | 71 +++ .../common/config/delta_subscription_state.cc | 85 ++-- .../common/config/delta_subscription_state.h | 58 ++- source/common/config/grpc_mux_impl.cc | 428 ++++++++---------- source/common/config/grpc_mux_impl.h | 255 +++++------ .../config/grpc_mux_subscription_impl.cc | 108 +++++ .../config/grpc_mux_subscription_impl.h | 50 ++ source/common/config/grpc_stream.h | 12 +- source/common/config/grpc_subscription_impl.h | 75 ++- source/common/config/new_grpc_mux_impl.cc | 228 ++++++++++ source/common/config/new_grpc_mux_impl.h | 120 +++++ source/common/config/pausable_ack_queue.h | 10 +- .../common/config/sotw_subscription_state.cc | 128 ------ .../common/config/sotw_subscription_state.h | 83 ---- .../config/subscription_factory_impl.cc | 71 +-- source/common/config/subscription_state.cc | 34 -- source/common/config/subscription_state.h | 79 ---- source/common/config/update_ack.h | 21 - source/common/config/watch_map.h | 3 +- .../common/upstream/cluster_manager_impl.cc | 36 +- test/common/config/BUILD | 45 +- .../config/delta_subscription_impl_test.cc | 20 +- .../config/delta_subscription_state_test.cc | 121 +++-- .../config/delta_subscription_test_harness.h | 18 +- test/common/config/grpc_mux_impl_test.cc | 413 ++++++++--------- .../config/grpc_subscription_impl_test.cc | 12 +- .../config/grpc_subscription_test_harness.h | 45 +- test/common/config/new_grpc_mux_impl_test.cc | 117 +++++ .../config/sotw_subscription_state_test.cc | 183 -------- test/common/config/subscription_impl_test.cc | 10 +- test/common/upstream/cds_api_impl_test.cc | 48 ++ test/config/utility.cc | 4 +- test/integration/ads_integration.h | 4 +- test/integration/ads_integration_test.cc | 19 +- test/integration/integration.cc | 87 ++-- test/integration/integration.h | 24 +- test/integration/rtds_integration_test.cc | 4 +- .../scoped_rds_integration_test.cc | 8 +- test/mocks/config/mocks.cc | 16 + test/mocks/config/mocks.h | 40 +- test/mocks/upstream/mocks.cc | 1 - tools/spelling_dictionary.txt | 6 - 48 files changed, 1845 insertions(+), 1661 deletions(-) rename source/common/config/{grpc_subscription_impl.cc => delta_subscription_impl.cc} (58%) create mode 100644 source/common/config/delta_subscription_impl.h create mode 100644 source/common/config/grpc_mux_subscription_impl.cc create mode 100644 source/common/config/grpc_mux_subscription_impl.h create mode 100644 source/common/config/new_grpc_mux_impl.cc create mode 100644 source/common/config/new_grpc_mux_impl.h delete mode 100644 source/common/config/sotw_subscription_state.cc delete mode 100644 source/common/config/sotw_subscription_state.h delete mode 100644 source/common/config/subscription_state.cc delete mode 100644 source/common/config/subscription_state.h delete mode 100644 source/common/config/update_ack.h create mode 100644 test/common/config/new_grpc_mux_impl_test.cc delete mode 100644 test/common/config/sotw_subscription_state_test.cc diff --git a/api/envoy/api/v2/core/config_source.proto b/api/envoy/api/v2/core/config_source.proto index ea86c2cd861a..b1e0fa582424 100644 --- a/api/envoy/api/v2/core/config_source.proto +++ b/api/envoy/api/v2/core/config_source.proto @@ -30,12 +30,15 @@ message ApiConfigSource { // the v2 protos is used. REST = 1; - // "State of the world" gRPC v2 API, using Discovery{Request,Response} protos. + // gRPC v2 API. GRPC = 2; - // "Delta" gRPC v2 API, using DeltaDiscovery{Request,Response} protos. - // Rather than sending Envoy the entire state with every update, the xDS server - // only sends what has changed since the last update. + // Using the delta xDS gRPC service, i.e. DeltaDiscovery{Request,Response} + // rather than Discovery{Request,Response}. Rather than sending Envoy the entire state + // with every update, the xDS server only sends what has changed since the last update. + // + // DELTA_GRPC is not yet entirely implemented! Initially, only CDS is available. + // Do not use for other xDSes. TODO(fredlas) update/remove this warning when appropriate. DELTA_GRPC = 3; } diff --git a/api/envoy/api/v3alpha/core/config_source.proto b/api/envoy/api/v3alpha/core/config_source.proto index d470fdef9af1..5da66d99fa83 100644 --- a/api/envoy/api/v3alpha/core/config_source.proto +++ b/api/envoy/api/v3alpha/core/config_source.proto @@ -30,12 +30,15 @@ message ApiConfigSource { // the v2 protos is used. REST = 1; - // "State of the world" gRPC v2 API, using Discovery{Request,Response} protos. + // gRPC v2 API. GRPC = 2; - // "Delta" gRPC v2 API, using DeltaDiscovery{Request,Response} protos. - // Rather than sending Envoy the entire state with every update, the xDS server - // only sends what has changed since the last update. + // Using the delta xDS gRPC service, i.e. DeltaDiscovery{Request,Response} + // rather than Discovery{Request,Response}. Rather than sending Envoy the entire state + // with every update, the xDS server only sends what has changed since the last update. + // + // DELTA_GRPC is not yet entirely implemented! Initially, only CDS is available. + // Do not use for other xDSes. TODO(fredlas) update/remove this warning when appropriate. DELTA_GRPC = 3; } diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index c07b0bab1646..8d65e28d58fa 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -25,6 +25,48 @@ struct ControlPlaneStats { ALL_CONTROL_PLANE_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) }; +// TODO(fredlas) redundant to SubscriptionCallbacks; remove this one. +class GrpcMuxCallbacks { +public: + virtual ~GrpcMuxCallbacks() = default; + + /** + * Called when a configuration update is received. + * @param resources vector of fetched resources corresponding to the configuration update. + * @param version_info update version. + * @throw EnvoyException with reason if the configuration is rejected. Otherwise the configuration + * is accepted. Accepted configurations have their version_info reflected in subsequent + * requests. + */ + virtual void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) PURE; + + /** + * Called when either the subscription is unable to fetch a config update or when onConfigUpdate + * invokes an exception. + * @param reason supplies the update failure reason. + * @param e supplies any exception data on why the fetch failed. May be nullptr. + */ + virtual void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) PURE; + + /** + * Obtain the "name" of a v2 API resource in a google.protobuf.Any, e.g. the route config name for + * a RouteConfiguration, based on the underlying resource type. + */ + virtual std::string resourceName(const ProtobufWkt::Any& resource) PURE; +}; + +/** + * Handle on an muxed gRPC subscription. The subscription is canceled on destruction. + */ +class GrpcMuxWatch { +public: + virtual ~GrpcMuxWatch() = default; +}; + +using GrpcMuxWatchPtr = std::unique_ptr; + struct Watch; /** @@ -40,12 +82,27 @@ class GrpcMux { */ virtual void start() PURE; + /** + * Start a configuration subscription asynchronously for some API type and resources. + * @param type_url type URL corresponding to xDS API, e.g. + * type.googleapis.com/envoy.api.v2.Cluster. + * @param resources set of resource names to watch for. If this is empty, then all + * resources for type_url will result in callbacks. + * @param callbacks the callbacks to be notified of configuration updates. These must be valid + * until GrpcMuxWatch is destroyed. + * @return GrpcMuxWatchPtr a handle to cancel the subscription with. E.g. when a cluster goes + * away, its EDS updates should be cancelled by destroying the GrpcMuxWatchPtr. + */ + virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, + const std::set& resources, + GrpcMuxCallbacks& callbacks) PURE; + /** * Pause discovery requests for a given API type. This is useful when we're processing an update * for LDS or CDS and don't want a flood of updates for RDS or EDS respectively. Discovery * requests may later be resumed with resume(). * @param type_url type URL corresponding to xDS API, e.g. - * type.googleapis.com/envoy.api.v2.Cluster + * type.googleapis.com/envoy.api.v2.Cluster. */ virtual void pause(const std::string& type_url) PURE; @@ -56,30 +113,18 @@ class GrpcMux { */ virtual void resume(const std::string& type_url) PURE; + // TODO(fredlas) PR #8478 will remove this. /** - * Registers a GrpcSubscription with the GrpcMux. 'watch' may be null (meaning this is an add), - * or it may be the Watch* previously returned by this function (which makes it an update). - * @param type_url type URL corresponding to xDS API e.g. type.googleapis.com/envoy.api.v2.Cluster - * @param watch the Watch* to be updated, or nullptr to add one. - * @param resources the set of resource names for 'watch' to start out interested in. If empty, - * 'watch' is treated as interested in *all* resources (of type type_url). - * @param callbacks the callbacks that receive updates for 'resources' when they arrive. - * @param init_fetch_timeout how long to wait for this new subscription's first update. Ignored - * unless the addOrUpdateWatch() call is the first for 'type_url'. - * @return Watch* the opaque watch token added or updated, to be used in future addOrUpdateWatch - * calls. + * Whether this GrpcMux is delta. + * @return bool whether this GrpcMux is delta. */ + virtual bool isDelta() const PURE; + + // For delta virtual Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, const std::set& resources, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout) PURE; - - /** - * Cleanup of a Watch* added by addOrUpdateWatch(). Receiving a Watch* from addOrUpdateWatch() - * makes you responsible for eventually invoking this cleanup. - * @param type_url type URL corresponding to xDS API e.g. type.googleapis.com/envoy.api.v2.Cluster - * @param watch the watch to be cleaned up. - */ virtual void removeWatch(const std::string& type_url, Watch* watch) PURE; /** @@ -89,12 +134,6 @@ class GrpcMux { * @return bool whether the API is paused. */ virtual bool paused(const std::string& type_url) const PURE; - - /** - * Passes through to all multiplexed SubscriptionStates. To be called when something - * definitive happens with the initial fetch: either an update is successfully received, - * or some sort of error happened.*/ - virtual void disableInitFetchTimeoutTimer() PURE; }; using GrpcMuxPtr = std::unique_ptr; diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 72cf3f94c991..bbce50a5a4d4 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -43,12 +43,12 @@ envoy_cc_library( ) envoy_cc_library( - name = "grpc_subscription_lib", - srcs = ["grpc_subscription_impl.cc"], - hdrs = ["grpc_subscription_impl.h"], + name = "delta_subscription_lib", + srcs = ["delta_subscription_impl.cc"], + hdrs = ["delta_subscription_impl.h"], deps = [ - ":grpc_mux_lib", ":grpc_stream_lib", + ":new_grpc_mux_lib", ":utility_lib", "//include/envoy/config:subscription_interface", "//include/envoy/grpc:async_client_interface", @@ -66,35 +66,15 @@ envoy_cc_library( srcs = ["delta_subscription_state.cc"], hdrs = ["delta_subscription_state.h"], deps = [ - ":subscription_state_lib", - "//source/common/grpc:common_lib", - "//source/common/protobuf", - "@envoy_api//envoy/api/v2:pkg_cc_proto", - ], -) - -envoy_cc_library( - name = "sotw_subscription_state_lib", - srcs = ["sotw_subscription_state.cc"], - hdrs = ["sotw_subscription_state.h"], - deps = [ - ":subscription_state_lib", - "//source/common/grpc:common_lib", - "//source/common/protobuf", - "@envoy_api//envoy/api/v2:pkg_cc_proto", - ], -) - -envoy_cc_library( - name = "subscription_state_lib", - srcs = ["subscription_state.cc"], - hdrs = ["subscription_state.h"], - deps = [ - ":update_ack_lib", + ":pausable_ack_queue_lib", "//include/envoy/config:subscription_interface", "//include/envoy/event:dispatcher_interface", - "//include/envoy/local_info:local_info_interface", + "//source/common/common:assert_lib", + "//source/common/common:backoff_lib", "//source/common/common:minimal_logger_lib", + "//source/common/common:token_bucket_impl_lib", + "//source/common/grpc:common_lib", + "//source/common/protobuf", "@envoy_api//envoy/api/v2:pkg_cc_proto", ], ) @@ -137,11 +117,54 @@ envoy_cc_library( name = "grpc_mux_lib", srcs = ["grpc_mux_impl.cc"], hdrs = ["grpc_mux_impl.h"], + deps = [ + ":grpc_stream_lib", + ":utility_lib", + "//include/envoy/config:grpc_mux_interface", + "//include/envoy/config:subscription_interface", + "//include/envoy/upstream:cluster_manager_interface", + "//source/common/common:minimal_logger_lib", + "//source/common/protobuf", + ], +) + +envoy_cc_library( + name = "grpc_mux_subscription_lib", + srcs = ["grpc_mux_subscription_impl.cc"], + hdrs = ["grpc_mux_subscription_impl.h"], + deps = [ + "//include/envoy/config:grpc_mux_interface", + "//include/envoy/config:subscription_interface", + "//include/envoy/event:dispatcher_interface", + "//source/common/common:assert_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/grpc:common_lib", + "//source/common/protobuf", + "@envoy_api//envoy/api/v2:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "grpc_subscription_lib", + hdrs = ["grpc_subscription_impl.h"], + deps = [ + ":grpc_mux_lib", + ":grpc_mux_subscription_lib", + "//include/envoy/config:subscription_interface", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/grpc:async_client_interface", + "@envoy_api//envoy/api/v2/core:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "new_grpc_mux_lib", + srcs = ["new_grpc_mux_impl.cc"], + hdrs = ["new_grpc_mux_impl.h"], deps = [ ":delta_subscription_state_lib", ":grpc_stream_lib", ":pausable_ack_queue_lib", - ":sotw_subscription_state_lib", ":watch_map_lib", "//include/envoy/event:dispatcher_interface", "//include/envoy/grpc:async_client_interface", @@ -189,8 +212,8 @@ envoy_cc_library( srcs = ["pausable_ack_queue.cc"], hdrs = ["pausable_ack_queue.h"], deps = [ - ":update_ack_lib", "//source/common/common:assert_lib", + "@envoy_api//envoy/api/v2:pkg_cc_proto", ], ) @@ -234,7 +257,9 @@ envoy_cc_library( srcs = ["subscription_factory_impl.cc"], hdrs = ["subscription_factory_impl.h"], deps = [ + ":delta_subscription_lib", ":filesystem_subscription_lib", + ":grpc_mux_subscription_lib", ":grpc_subscription_lib", ":http_subscription_lib", ":type_to_endpoint_lib", @@ -259,12 +284,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "update_ack_lib", - hdrs = ["update_ack.h"], - deps = ["@envoy_api//envoy/api/v2:pkg_cc_proto"], -) - envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], diff --git a/source/common/config/README.md b/source/common/config/README.md index cfcecd7678a1..3135bd56bdaa 100644 --- a/source/common/config/README.md +++ b/source/common/config/README.md @@ -9,19 +9,24 @@ you can mostly forget the filesystem/REST/gRPC distinction, and you can especially forget about the gRPC flavors. All of that is specified in the bootstrap config, which is read and put into action by ClusterManagerImpl. -## If you are working on Envoy's gRPC xDS client logic itself, read on. +Note that there can be multiple active gRPC subscriptions for a single resource +type. This concept is called "resource watches". If one EDS subscription +subscribes to X and Y, and another subscribes to Y and Z, the underlying +subscription logic will maintain a subscription to the union: X Y and Z. Updates +to X will be delivered to the first object, Y to both, Z to the second. This +logic is implemented by WatchMap. + +### If you are working on Envoy's gRPC xDS client logic itself, read on. When using gRPC, xDS has two pairs of options: aggregated/non-aggregated, and delta/state-of-the-world updates. All four combinations of these are usable. -## Aggregated (ADS) vs not (xDS) - "Aggregated" means that EDS, CDS, etc resources are all carried by the same gRPC stream. For Envoy's implementation of xDS client logic, there is effectively no difference between aggregated xDS and non-aggregated: they both use the same request/response protos. The non-aggregated case is handled by running the aggregated logic, and just happening to only have 1 -xDS subscription type to "aggregate", i.e., GrpcMux only has one SubscriptionState -entry in its map. +xDS subscription type to "aggregate", i.e., NewGrpcMuxImpl only has one +DeltaSubscriptionState entry in its map. However, to the config server, there is a huge difference: when using ADS (caused by the user providing an ads_config in the bootstrap config), the gRPC client sets @@ -29,31 +34,14 @@ its method string to {Delta,Stream}AggregatedResources, as opposed to {Delta,Str {Delta,Stream}Routes, etc. So, despite using the same request/response protos, and having identical client code, they're actually different gRPC services. -## Delta vs state-of-the-world (SotW) - -Delta vs state-of-the-world is a question of wire format and protocol behavior. -The protos in question are named [Delta]Discovery{Request,Response}. GrpcMux can work -with either pair. Almost all GrpcMux logic is in the shared GrpcMuxImpl base class; -SotwGrpcMux and DeltaGrpcMux exist to be adapters for the specific protobuf types, since -protobufs are not amenable to polymorphism. - -All delta/SotW specific logic is handled by GrpcMux and SubscriptionState. GrpcSubscriptionImpl -simply holds a shared_ptr to a GrpcMux interface; it has no need to know about delta vs SotW. +Delta vs state-of-the-world is a question of wire format: the protos in question are named +[Delta]Discovery{Request,Response}. That is what the GrpcMux interface is useful for: its +NewGrpcMuxImpl (TODO may be renamed) implementation works with DeltaDiscovery{Request,Response} and has +delta-specific logic; its GrpxMuxImpl implementation (TODO will be merged into NewGrpcMuxImpl) works with Discovery{Request,Response} +and has SotW-specific logic. Both the delta and SotW Subscription implementations (TODO will be merged) hold a shared_ptr. +The shared_ptr allows for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. ![xDS_code_diagram](xDS_code_diagram.png) -The orange flow does not necessarily have to happen in response to the blue flow; there can be -spontaneous updates. ACKs are not shown in this diagram; they are also carried by the -[Delta]DiscoveryRequest protos. - -What does GrpcMux even do in this diagram? Just own things and pass through function calls? -Answer: 1) it sequences the requests and ACKs that the various type_urls send, 2) it handles the -protobuf vs polymorphism impedance mismatch, allowing all delta-vs-SotW-agnostic code -to be reused. - -Note that there can be multiple active gRPC subscriptions for a single resource -type. This concept is called "resource watches". If one EDS subscription -subscribes to X and Y, and another subscribes to Y and Z, the underlying -subscription logic will maintain a subscription to the union: X Y and Z. Updates -to X will be delivered to the first object, Y to both, Z to the second. This -logic is implemented by WatchMap. +Note that the orange flow does not necessarily have to happen in response to the blue flow; there can be spontaneous updates. ACKs are not shown in this diagram; they are also carred by the [Delta]DiscoveryRequest protos. +What does GrpcXdsContext even do in this diagram? Just own things and pass through function calls? Answer: it sequences the requests and ACKs that the various type_urls send. diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc similarity index 58% rename from source/common/config/grpc_subscription_impl.cc rename to source/common/config/delta_subscription_impl.cc index 77f003b061b5..ed34dac39bea 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -1,68 +1,66 @@ -#include "common/config/grpc_subscription_impl.h" +#include "common/config/delta_subscription_impl.h" namespace Envoy { namespace Config { -GrpcSubscriptionImpl::GrpcSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, absl::string_view type_url, - SubscriptionCallbacks& callbacks, - SubscriptionStats stats, - std::chrono::milliseconds init_fetch_timeout, - bool is_aggregated) - : grpc_mux_(std::move(grpc_mux)), type_url_(type_url), callbacks_(callbacks), stats_(stats), +DeltaSubscriptionImpl::DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, + SubscriptionCallbacks& callbacks, + SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout, + bool is_aggregated) + : context_(std::move(context)), type_url_(type_url), callbacks_(callbacks), stats_(stats), init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated) {} -GrpcSubscriptionImpl::~GrpcSubscriptionImpl() { +DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { if (watch_) { - grpc_mux_->removeWatch(type_url_, watch_); + context_->removeWatch(type_url_, watch_); } } -void GrpcSubscriptionImpl::pause() { grpc_mux_->pause(type_url_); } +void DeltaSubscriptionImpl::pause() { context_->pause(type_url_); } -void GrpcSubscriptionImpl::resume() { grpc_mux_->resume(type_url_); } +void DeltaSubscriptionImpl::resume() { context_->resume(type_url_); } -// Config::Subscription -void GrpcSubscriptionImpl::start(const std::set& resources) { +// Config::DeltaSubscription +void DeltaSubscriptionImpl::start(const std::set& resources) { // ADS initial request batching relies on the users of the GrpcMux *not* calling start on it, // whereas non-ADS xDS users must call it themselves. if (!is_aggregated_) { - grpc_mux_->start(); + context_->start(); } - watch_ = grpc_mux_->addOrUpdateWatch(type_url_, watch_, resources, *this, init_fetch_timeout_); + watch_ = context_->addOrUpdateWatch(type_url_, watch_, resources, *this, init_fetch_timeout_); stats_.update_attempt_.inc(); } -void GrpcSubscriptionImpl::updateResourceInterest( +void DeltaSubscriptionImpl::updateResourceInterest( const std::set& update_to_these_names) { - watch_ = grpc_mux_->addOrUpdateWatch(type_url_, watch_, update_to_these_names, *this, - init_fetch_timeout_); + watch_ = context_->addOrUpdateWatch(type_url_, watch_, update_to_these_names, *this, + init_fetch_timeout_); stats_.update_attempt_.inc(); } // Config::SubscriptionCallbacks -void GrpcSubscriptionImpl::onConfigUpdate( +void DeltaSubscriptionImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { stats_.update_attempt_.inc(); callbacks_.onConfigUpdate(resources, version_info); - grpc_mux_->disableInitFetchTimeoutTimer(); stats_.update_success_.inc(); stats_.version_.set(HashUtil::xxHash64(version_info)); } -void GrpcSubscriptionImpl::onConfigUpdate( +void DeltaSubscriptionImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { stats_.update_attempt_.inc(); callbacks_.onConfigUpdate(added_resources, removed_resources, system_version_info); - grpc_mux_->disableInitFetchTimeoutTimer(); stats_.update_success_.inc(); stats_.version_.set(HashUtil::xxHash64(system_version_info)); } -void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, - const EnvoyException* e) { +void DeltaSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, + const EnvoyException* e) { switch (reason) { case Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure: // This is a gRPC-stream-level establishment failure, not an xDS-protocol-level failure. @@ -75,23 +73,21 @@ void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason break; case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: stats_.init_fetch_timeout_.inc(); - grpc_mux_->disableInitFetchTimeoutTimer(); - callbacks_.onConfigUpdateFailed(reason, e); // TODO(fredlas) remove; it only makes sense to count start() and updateResourceInterest() // as attempts. stats_.update_attempt_.inc(); + callbacks_.onConfigUpdateFailed(reason, e); break; case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: // We expect Envoy exception to be thrown when update is rejected. ASSERT(e != nullptr); - grpc_mux_->disableInitFetchTimeoutTimer(); stats_.update_rejected_.inc(); callbacks_.onConfigUpdateFailed(reason, e); break; } } -std::string GrpcSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { +std::string DeltaSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { return callbacks_.resourceName(resource); } diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h new file mode 100644 index 000000000000..b45be7c4da85 --- /dev/null +++ b/source/common/config/delta_subscription_impl.h @@ -0,0 +1,71 @@ +#pragma once + +#include "envoy/config/subscription.h" + +#include "common/config/new_grpc_mux_impl.h" +#include "common/config/utility.h" + +namespace Envoy { +namespace Config { + +// DeltaSubscriptionImpl provides a top-level interface to the Envoy's gRPC communication with +// an xDS server, for use by the various xDS users within Envoy. It is built around a (shared) +// NewGrpcMuxImpl, and the further machinery underlying that. An xDS user indicates interest in +// various resources via start() and updateResourceInterest(). It receives updates to those +// resources via the SubscriptionCallbacks it provides. Multiple users can each have their own +// Subscription object for the same type_url; NewGrpcMuxImpl maintains a subscription to the +// union of interested resources, and delivers to the users just the resource updates that they +// are "watching" for. +// +// DeltaSubscriptionImpl and NewGrpcMuxImpl are both built to provide both regular xDS and ADS, +// distinguished by whether multiple DeltaSubscriptionImpls are sharing a single +// NewGrpcMuxImpl. (And by the gRPC method string, but that's taken care of over in +// SubscriptionFactory). +// +// Why does DeltaSubscriptionImpl itself implement the SubscriptionCallbacks interface? So that it +// can write to SubscriptionStats (which needs to live out here in the DeltaSubscriptionImpl) upon a +// config update. The idea is, DeltaSubscriptionImpl presents itself to WatchMap as the +// SubscriptionCallbacks, and then passes (after incrementing stats) all callbacks through to +// callbacks_, which are the real SubscriptionCallbacks. +class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks { +public: + // is_aggregated: whether the underlying mux/context is providing ADS to us and others, or whether + // it's all ours. The practical difference is that we ourselves must call start() on it only in + // the latter case. + DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout, bool is_aggregated); + ~DeltaSubscriptionImpl() override; + + void pause(); + void resume(); + + // Config::Subscription + void start(const std::set& resource_names) override; + void updateResourceInterest(const std::set& update_to_these_names) override; + + // Config::SubscriptionCallbacks (all pass through to callbacks_!) + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override; + void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override; + void onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) override; + std::string resourceName(const ProtobufWkt::Any& resource) override; + + GrpcMuxSharedPtr getContextForTest() { return context_; } + +private: + GrpcMuxSharedPtr context_; + const std::string type_url_; + SubscriptionCallbacks& callbacks_; + SubscriptionStats stats_; + // NOTE: if another subscription of the same type_url has already been started, this value will be + // ignored in favor of the other subscription's. + std::chrono::milliseconds init_fetch_timeout_; + Watch* watch_{}; + const bool is_aggregated_; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index 8eccd7343aa5..8fdfdca0ffca 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -8,23 +8,18 @@ namespace Config { DeltaSubscriptionState::DeltaSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher) - : SubscriptionState(std::move(type_url), callbacks, init_fetch_timeout, dispatcher) {} - -DeltaSubscriptionState::~DeltaSubscriptionState() = default; - -DeltaSubscriptionStateFactory::DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher) - : dispatcher_(dispatcher) {} - -DeltaSubscriptionStateFactory::~DeltaSubscriptionStateFactory() = default; - -std::unique_ptr -DeltaSubscriptionStateFactory::makeSubscriptionState(const std::string& type_url, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - return std::make_unique(type_url, callbacks, init_fetch_timeout, - dispatcher_); + : type_url_(std::move(type_url)), callbacks_(callbacks), local_info_(local_info), + init_fetch_timeout_(init_fetch_timeout) { + if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { + init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { + ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); + callbacks_.onConfigUpdateFailed(ConfigUpdateFailureReason::FetchTimedout, nullptr); + }); + init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); + } } void DeltaSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, @@ -56,13 +51,13 @@ bool DeltaSubscriptionState::subscriptionUpdatePending() const { !any_request_sent_yet_in_current_stream_; } -UpdateAck DeltaSubscriptionState::handleResponse(const void* response_proto_ptr) { - auto* response = static_cast(response_proto_ptr); +UpdateAck +DeltaSubscriptionState::handleResponse(const envoy::api::v2::DeltaDiscoveryResponse& message) { // We *always* copy the response's nonce into the next request, even if we're going to make that // request a NACK by setting error_detail. - UpdateAck ack(response->nonce(), type_url()); + UpdateAck ack(message.nonce(), type_url_); try { - handleGoodResponse(*response); + handleGoodResponse(message); } catch (const EnvoyException& e) { handleBadResponse(e, ack); } @@ -91,8 +86,8 @@ void DeltaSubscriptionState::handleGoodResponse( fmt::format("duplicate name {} found in the union of added+removed resources", name)); } } - callbacks().onConfigUpdate(message.resources(), message.removed_resources(), - message.system_version_info()); + callbacks_.onConfigUpdate(message.resources(), message.removed_resources(), + message.system_version_info()); for (const auto& resource : message.resources()) { setResourceVersion(resource.name(), resource.version()); } @@ -105,11 +100,11 @@ void DeltaSubscriptionState::handleGoodResponse( // initial_resource_versions messages, but will remind us to explicitly tell the server "I'm // cancelling my subscription" when we lose interest. for (const auto& resource_name : message.removed_resources()) { - if (resource_versions_.find(resource_name) != resource_versions_.end()) { + if (resource_names_.find(resource_name) != resource_names_.end()) { setResourceWaitingForServer(resource_name); } } - ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url(), + ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url_, message.resources().size(), message.removed_resources().size()); } @@ -118,19 +113,17 @@ void DeltaSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAc ack.error_detail_.set_code(Grpc::Status::WellKnownGrpcStatus::Internal); ack.error_detail_.set_message(e.what()); disableInitFetchTimeoutTimer(); - ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url(), e.what()); - callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); + ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } void DeltaSubscriptionState::handleEstablishmentFailure() { - callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, - nullptr); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + nullptr); } -envoy::api::v2::DeltaDiscoveryRequest* DeltaSubscriptionState::getNextRequestInternal() { - auto* request = new envoy::api::v2::DeltaDiscoveryRequest; - request->set_type_url(type_url()); - +envoy::api::v2::DeltaDiscoveryRequest DeltaSubscriptionState::getNextRequestAckless() { + envoy::api::v2::DeltaDiscoveryRequest request; if (!any_request_sent_yet_in_current_stream_) { any_request_sent_yet_in_current_stream_ = true; // initial_resource_versions "must be populated for first request in a stream". @@ -141,7 +134,7 @@ envoy::api::v2::DeltaDiscoveryRequest* DeltaSubscriptionState::getNextRequestInt // Resources we are interested in, but are still waiting to get any version of from the // server, do not belong in initial_resource_versions. (But do belong in new subscriptions!) if (!resource.second.waitingForServer()) { - (*request->mutable_initial_resource_versions())[resource.first] = resource.second.version(); + (*request.mutable_initial_resource_versions())[resource.first] = resource.second.version(); } // As mentioned above, fill resource_names_subscribe with everything, including names we // have yet to receive any resource for. @@ -149,40 +142,50 @@ envoy::api::v2::DeltaDiscoveryRequest* DeltaSubscriptionState::getNextRequestInt } names_removed_.clear(); } - std::copy(names_added_.begin(), names_added_.end(), - Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names_subscribe())); + Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_subscribe())); std::copy(names_removed_.begin(), names_removed_.end(), - Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names_unsubscribe())); + Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_unsubscribe())); names_added_.clear(); names_removed_.clear(); + request.set_type_url(type_url_); + request.mutable_node()->MergeFrom(local_info_.node()); return request; } -void* DeltaSubscriptionState::getNextRequestAckless() { return getNextRequestInternal(); } - -void* DeltaSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { - envoy::api::v2::DeltaDiscoveryRequest* request = getNextRequestInternal(); - request->set_response_nonce(ack.nonce_); +envoy::api::v2::DeltaDiscoveryRequest +DeltaSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { + envoy::api::v2::DeltaDiscoveryRequest request = getNextRequestAckless(); + request.set_response_nonce(ack.nonce_); if (ack.error_detail_.code() != Grpc::Status::WellKnownGrpcStatus::Ok) { // Don't needlessly make the field present-but-empty if status is ok. - request->mutable_error_detail()->CopyFrom(ack.error_detail_); + request.mutable_error_detail()->CopyFrom(ack.error_detail_); } return request; } +void DeltaSubscriptionState::disableInitFetchTimeoutTimer() { + if (init_fetch_timeout_timer_) { + init_fetch_timeout_timer_->disableTimer(); + init_fetch_timeout_timer_.reset(); + } +} + void DeltaSubscriptionState::setResourceVersion(const std::string& resource_name, const std::string& resource_version) { resource_versions_[resource_name] = ResourceVersion(resource_version); + resource_names_.insert(resource_name); } void DeltaSubscriptionState::setResourceWaitingForServer(const std::string& resource_name) { resource_versions_[resource_name] = ResourceVersion(); + resource_names_.insert(resource_name); } void DeltaSubscriptionState::setLostInterestInResource(const std::string& resource_name) { resource_versions_.erase(resource_name); + resource_names_.erase(resource_name); } } // namespace Config diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 31c34c0322d2..6477bdd01496 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -1,56 +1,55 @@ #pragma once #include "envoy/api/v2/discovery.pb.h" +#include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" #include "envoy/grpc/status.h" +#include "envoy/local_info/local_info.h" #include "common/common/assert.h" #include "common/common/logger.h" -#include "common/config/subscription_state.h" - -#include "absl/types/optional.h" +#include "common/config/pausable_ack_queue.h" namespace Envoy { namespace Config { -// Tracks the state of a delta xDS-over-gRPC protocol session. -class DeltaSubscriptionState : public SubscriptionState { +// Tracks the xDS protocol state of an individual ongoing delta xDS session, i.e. a single type_url. +// There can be multiple DeltaSubscriptionStates active. They will always all be +// blissfully unaware of each other's existence, even when their messages are +// being multiplexed together by ADS. +class DeltaSubscriptionState : public Logger::Loggable { public: - // Note that, outside of tests, we expect callbacks to always be a WatchMap. DeltaSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher); - ~DeltaSubscriptionState() override; // Update which resources we're interested in subscribing to. void updateSubscriptionInterest(const std::set& cur_added, - const std::set& cur_removed) override; + const std::set& cur_removed); // Whether there was a change in our subscription interest we have yet to inform the server of. - bool subscriptionUpdatePending() const override; + bool subscriptionUpdatePending() const; - void markStreamFresh() override { any_request_sent_yet_in_current_stream_ = false; } + void markStreamFresh() { any_request_sent_yet_in_current_stream_ = false; } - // message is expected to be a envoy::api::v2::DeltaDiscoveryResponse. - UpdateAck handleResponse(const void* response_proto_ptr) override; + UpdateAck handleResponse(const envoy::api::v2::DeltaDiscoveryResponse& message); - void handleEstablishmentFailure() override; + void handleEstablishmentFailure(); // Returns the next gRPC request proto to be sent off to the server, based on this object's // understanding of the current protocol state, and new resources that Envoy wants to request. - // Returns a new'd pointer, meant to be owned by the caller. - void* getNextRequestAckless() override; - // The WithAck version first calls the Ackless version, then adds in the passed-in ack. - // Returns a new'd pointer, meant to be owned by the caller. - void* getNextRequestWithAck(const UpdateAck& ack) override; + envoy::api::v2::DeltaDiscoveryRequest getNextRequestAckless(); + // The WithAck version first calls the Ack-less version, then adds in the passed-in ack. + envoy::api::v2::DeltaDiscoveryRequest getNextRequestWithAck(const UpdateAck& ack); DeltaSubscriptionState(const DeltaSubscriptionState&) = delete; DeltaSubscriptionState& operator=(const DeltaSubscriptionState&) = delete; private: - // Returns a new'd pointer, meant to be owned by the caller. - envoy::api::v2::DeltaDiscoveryRequest* getNextRequestInternal(); void handleGoodResponse(const envoy::api::v2::DeltaDiscoveryResponse& message); void handleBadResponse(const EnvoyException& e, UpdateAck& ack); + void disableInitFetchTimeoutTimer(); class ResourceVersion { public: @@ -85,6 +84,13 @@ class DeltaSubscriptionState : public SubscriptionState { // iterator into just its keys, e.g. for use in std::set_difference. std::set resource_names_; + const std::string type_url_; + // callbacks_ is expected to be a WatchMap. + SubscriptionCallbacks& callbacks_; + const LocalInfo::LocalInfo& local_info_; + std::chrono::milliseconds init_fetch_timeout_; + Event::TimerPtr init_fetch_timeout_timer_; + bool any_request_sent_yet_in_current_stream_{}; // Tracks changes in our subscription interest since the previous DeltaDiscoveryRequest we sent. @@ -94,17 +100,5 @@ class DeltaSubscriptionState : public SubscriptionState { std::set names_removed_; }; -class DeltaSubscriptionStateFactory : public SubscriptionStateFactory { -public: - DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher); - ~DeltaSubscriptionStateFactory() override; - std::unique_ptr - makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) override; - -private: - Event::Dispatcher& dispatcher_; -}; - } // namespace Config } // namespace Envoy diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 93662e0e21c4..3246dfbbd189 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -1,297 +1,249 @@ #include "common/config/grpc_mux_impl.h" -#include "common/common/assert.h" -#include "common/common/backoff_strategy.h" -#include "common/common/token_bucket_impl.h" +#include + #include "common/config/utility.h" #include "common/protobuf/protobuf.h" -#include "common/protobuf/utility.h" namespace Envoy { namespace Config { -GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory, - bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info) - : subscription_state_factory_(std::move(subscription_state_factory)), - skip_subsequent_node_(skip_subsequent_node), local_info_(local_info) { +GrpcMuxImpl::GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, + Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node) + : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings), + local_info_(local_info), skip_subsequent_node_(skip_subsequent_node), + first_stream_request_(true) { Config::Utility::checkLocalInfo("ads", local_info); } -Watch* GrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, - const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - if (watch == nullptr) { - return addWatch(type_url, resources, callbacks, init_fetch_timeout); - } else { - updateWatch(type_url, watch, resources); - return watch; +GrpcMuxImpl::~GrpcMuxImpl() { + for (const auto& api_state : api_state_) { + for (auto watch : api_state.second.watches_) { + watch->clear(); + } } } -void GrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { - updateWatch(type_url, watch, {}); - watchMapFor(type_url).removeWatch(watch); -} - -void GrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } - -void GrpcMuxImpl::resume(const std::string& type_url) { - pausable_ack_queue_.resume(type_url); - trySendDiscoveryRequests(); -} - -bool GrpcMuxImpl::paused(const std::string& type_url) const { - return pausable_ack_queue_.paused(type_url); -} +void GrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } -void GrpcMuxImpl::genericHandleResponse(const std::string& type_url, - const void* response_proto_ptr) { - auto sub = subscriptions_.find(type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, - "The server sent an xDS response proto with type_url {}, which we have " - "not subscribed to. Ignoring.", - type_url); - return; +void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { + if (!grpc_stream_.grpcStreamAvailable()) { + ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", type_url); + return; // Drop this request; the reconnect will enqueue a new one. } - pausable_ack_queue_.push(sub->second->handleResponse(response_proto_ptr)); - trySendDiscoveryRequests(); -} - -void GrpcMuxImpl::start() { establishGrpcStream(); } -void GrpcMuxImpl::handleEstablishedStream() { - for (auto& sub : subscriptions_) { - sub.second->markStreamFresh(); + ApiState& api_state = api_state_[type_url]; + if (api_state.paused_) { + ENVOY_LOG(trace, "API {} paused during sendDiscoveryRequest(), setting pending.", type_url); + api_state.pending_ = true; + return; // Drop this request; the unpause will enqueue a new one. } - set_any_request_sent_yet_in_current_stream(false); - trySendDiscoveryRequests(); -} -void GrpcMuxImpl::disableInitFetchTimeoutTimer() { - for (auto& sub : subscriptions_) { - sub.second->disableInitFetchTimeoutTimer(); - } -} + auto& request = api_state.request_; + request.mutable_resource_names()->Clear(); -void GrpcMuxImpl::handleStreamEstablishmentFailure() { - // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately - // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a - // crash, the iteration needs to dance around a little: collect pointers to all - // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are - // now more SubscriptionStates. - absl::flat_hash_map all_subscribed; - absl::flat_hash_map already_called; - do { - for (auto& sub : subscriptions_) { - all_subscribed[sub.first] = sub.second.get(); - } - for (auto& sub : all_subscribed) { - if (already_called.insert(sub).second) { // insert succeeded ==> not already called - sub.second->handleEstablishmentFailure(); + // Maintain a set to avoid dupes. + std::unordered_set resources; + for (const auto* watch : api_state.watches_) { + for (const std::string& resource : watch->resources_) { + if (resources.count(resource) == 0) { + resources.emplace(resource); + request.add_resource_names(resource); } } - } while (all_subscribed.size() != subscriptions_.size()); -} + } -Watch* GrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - auto watch_map = watch_maps_.find(type_url); - if (watch_map == watch_maps_.end()) { - // We don't yet have a subscription for type_url! Make one! - watch_map = watch_maps_.emplace(type_url, std::make_unique()).first; - subscriptions_.emplace(type_url, subscription_state_factory_->makeSubscriptionState( - type_url, *watch_maps_[type_url], init_fetch_timeout)); - subscription_ordering_.emplace_back(type_url); + if (skip_subsequent_node_ && !first_stream_request_) { + request.clear_node(); } + ENVOY_LOG(trace, "Sending DiscoveryRequest for {}: {}", type_url, request.DebugString()); + grpc_stream_.sendMessage(request); + first_stream_request_ = false; - Watch* watch = watch_map->second->addWatch(callbacks); - // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. - updateWatch(type_url, watch, resources); - return watch; + // clear error_detail after the request is sent if it exists. + if (api_state_[type_url].request_.has_error_detail()) { + api_state_[type_url].request_.clear_error_detail(); + } } -// Updates the list of resource names watched by the given watch. If an added name is new across -// the whole subscription, or if a removed name has no other watch interested in it, then the -// subscription will enqueue and attempt to send an appropriate discovery request. -void GrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources) { - ASSERT(watch != nullptr); - SubscriptionState& sub = subscriptionStateFor(type_url); - WatchMap& watch_map = watchMapFor(type_url); - - auto added_removed = watch_map.updateWatchInterest(watch, resources); - sub.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); - - // Tell the server about our change in interest, if any. - if (sub.subscriptionUpdatePending()) { - trySendDiscoveryRequests(); +GrpcMuxWatchPtr GrpcMuxImpl::subscribe(const std::string& type_url, + const std::set& resources, + GrpcMuxCallbacks& callbacks) { + auto watch = + std::unique_ptr(new GrpcMuxWatchImpl(resources, callbacks, type_url, *this)); + ENVOY_LOG(debug, "gRPC mux subscribe for " + type_url); + + // Lazily kick off the requests based on first subscription. This has the + // convenient side-effect that we order messages on the channel based on + // Envoy's internal dependency ordering. + // TODO(gsagula): move TokenBucketImpl params to a config. + if (!api_state_[type_url].subscribed_) { + api_state_[type_url].request_.set_type_url(type_url); + api_state_[type_url].request_.mutable_node()->MergeFrom(local_info_.node()); + api_state_[type_url].subscribed_ = true; + subscriptions_.emplace_back(type_url); } -} -SubscriptionState& GrpcMuxImpl::subscriptionStateFor(const std::string& type_url) { - auto sub = subscriptions_.find(type_url); - RELEASE_ASSERT(sub != subscriptions_.end(), - fmt::format("Tried to look up SubscriptionState for non-existent subscription {}.", - type_url)); - return *sub->second; + // This will send an updated request on each subscription. + // TODO(htuch): For RDS/EDS, this will generate a new DiscoveryRequest on each resource we added. + // Consider in the future adding some kind of collation/batching during CDS/LDS updates so that we + // only send a single RDS/EDS update after the CDS/LDS update. + queueDiscoveryRequest(type_url); + + return watch; } -WatchMap& GrpcMuxImpl::watchMapFor(const std::string& type_url) { - auto watch_map = watch_maps_.find(type_url); - RELEASE_ASSERT( - watch_map != watch_maps_.end(), - fmt::format("Tried to look up WatchMap for non-existent subscription {}.", type_url)); - return *watch_map->second; +void GrpcMuxImpl::pause(const std::string& type_url) { + ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url); + ApiState& api_state = api_state_[type_url]; + ASSERT(!api_state.paused_); + ASSERT(!api_state.pending_); + api_state.paused_ = true; } -void GrpcMuxImpl::trySendDiscoveryRequests() { - while (true) { - // Do any of our subscriptions even want to send a request? - absl::optional request_type_if_any = whoWantsToSendDiscoveryRequest(); - if (!request_type_if_any.has_value()) { - break; - } - // If so, which one (by type_url)? - std::string next_request_type_url = request_type_if_any.value(); - SubscriptionState& sub = subscriptionStateFor(next_request_type_url); - // Try again later if paused/rate limited/stream down. - if (!canSendDiscoveryRequest(next_request_type_url)) { - break; - } - // Get our subscription state to generate the appropriate discovery request, and send. - if (!pausable_ack_queue_.empty()) { - // Because ACKs take precedence over plain requests, if there is anything in the queue, it's - // safe to assume it's of the type_url that we're wanting to send. - // - // getNextRequestWithAck() returns a raw unowned pointer, which sendGrpcMessage deletes. - sendGrpcMessage(sub.getNextRequestWithAck(pausable_ack_queue_.front())); - pausable_ack_queue_.pop(); - } else { - // getNextRequestAckless() returns a raw unowned pointer, which sendGrpcMessage deletes. - sendGrpcMessage(sub.getNextRequestAckless()); - } +void GrpcMuxImpl::resume(const std::string& type_url) { + ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url); + ApiState& api_state = api_state_[type_url]; + ASSERT(api_state.paused_); + api_state.paused_ = false; + + if (api_state.pending_) { + ASSERT(api_state.subscribed_); + queueDiscoveryRequest(type_url); + api_state.pending_ = false; } - maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); } -// Checks whether external conditions allow sending a discovery request. (Does not check -// whether we *want* to send a discovery request). -bool GrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { - RELEASE_ASSERT( - !pausable_ack_queue_.paused(type_url), - fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " - "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", - type_url)); - - if (!grpcStreamAvailable()) { - ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); - return false; - } else if (!rateLimitAllowsDrain()) { - ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); +bool GrpcMuxImpl::paused(const std::string& type_url) const { + auto entry = api_state_.find(type_url); + if (entry == api_state_.end()) { return false; } - return true; + return entry->second.paused_; } -// Checks whether we have something to say in a discovery request, which can be an ACK and/or -// a subscription update. (Does not check whether we *can* send that discovery request). -// Returns the type_url we should send the discovery request for (if any). -// First, prioritizes ACKs over non-ACK subscription interest updates. -// Then, prioritizes non-ACK updates in the order the various types -// of subscriptions were activated. -absl::optional GrpcMuxImpl::whoWantsToSendDiscoveryRequest() { - // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose - // type_url from pausable_ack_queue_ if possible, before looking at pending updates. - if (!pausable_ack_queue_.empty()) { - return pausable_ack_queue_.front().type_url_; +void GrpcMuxImpl::onDiscoveryResponse( + std::unique_ptr&& message) { + const std::string& type_url = message->type_url(); + ENVOY_LOG(debug, "Received gRPC message for {} at version {}", type_url, message->version_info()); + if (api_state_.count(type_url) == 0) { + ENVOY_LOG(warn, "Ignoring the message for type URL {} as it has no current subscribers.", + type_url); + // TODO(yuval-k): This should never happen. consider dropping the stream as this is a + // protocol violation + return; } - // If we're looking to send multiple non-ACK requests, send them in the order that their - // subscriptions were initiated. - for (const auto& sub_type : subscription_ordering_) { - SubscriptionState& sub = subscriptionStateFor(sub_type); - if (sub.subscriptionUpdatePending() && !pausable_ack_queue_.paused(sub_type)) { - return sub_type; + if (api_state_[type_url].watches_.empty()) { + // update the nonce as we are processing this response. + api_state_[type_url].request_.set_response_nonce(message->nonce()); + if (message->resources().empty()) { + // No watches and no resources. This can happen when envoy unregisters from a + // resource that's removed from the server as well. For example, a deleted cluster + // triggers un-watching the ClusterLoadAssignment watch, and at the same time the + // xDS server sends an empty list of ClusterLoadAssignment resources. we'll accept + // this update. no need to send a discovery request, as we don't watch for anything. + api_state_[type_url].request_.set_version_info(message->version_info()); + } else { + // No watches and we have resources - this should not happen. send a NACK (by not + // updating the version). + ENVOY_LOG(warn, "Ignoring unwatched type URL {}", type_url); + queueDiscoveryRequest(type_url); + } + return; + } + try { + // To avoid O(n^2) explosion (e.g. when we have 1000s of EDS watches), we + // build a map here from resource name to resource and then walk watches_. + // We have to walk all watches (and need an efficient map as a result) to + // ensure we deliver empty config updates when a resource is dropped. + std::unordered_map resources; + GrpcMuxCallbacks& callbacks = api_state_[type_url].watches_.front()->callbacks_; + for (const auto& resource : message->resources()) { + if (type_url != resource.type_url()) { + throw EnvoyException( + fmt::format("{} does not match the message-wide type URL {} in DiscoveryResponse {}", + resource.type_url(), type_url, message->DebugString())); + } + const std::string resource_name = callbacks.resourceName(resource); + resources.emplace(resource_name, resource); + } + for (auto watch : api_state_[type_url].watches_) { + // onConfigUpdate should be called in all cases for single watch xDS (Cluster and + // Listener) even if the message does not have resources so that update_empty stat + // is properly incremented and state-of-the-world semantics are maintained. + if (watch->resources_.empty()) { + watch->callbacks_.onConfigUpdate(message->resources(), message->version_info()); + continue; + } + Protobuf::RepeatedPtrField found_resources; + for (const auto& watched_resource_name : watch->resources_) { + auto it = resources.find(watched_resource_name); + if (it != resources.end()) { + found_resources.Add()->MergeFrom(it->second); + } + } + // onConfigUpdate should be called only on watches(clusters/routes) that have + // updates in the message for EDS/RDS. + if (!found_resources.empty()) { + watch->callbacks_.onConfigUpdate(found_resources, message->version_info()); + } + } + // TODO(mattklein123): In the future if we start tracking per-resource versions, we + // would do that tracking here. + api_state_[type_url].request_.set_version_info(message->version_info()); + } catch (const EnvoyException& e) { + for (auto watch : api_state_[type_url].watches_) { + watch->callbacks_.onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } + ::google::rpc::Status* error_detail = api_state_[type_url].request_.mutable_error_detail(); + error_detail->set_code(Grpc::Status::WellKnownGrpcStatus::Internal); + error_detail->set_message(e.what()); } - return absl::nullopt; + api_state_[type_url].request_.set_response_nonce(message->nonce()); + queueDiscoveryRequest(type_url); } -// Delta- and SotW-specific concrete subclasses: -GrpcMuxDelta::GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) - : GrpcMuxImpl(std::make_unique(dispatcher), skip_subsequent_node, - local_info), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} +void GrpcMuxImpl::onWriteable() { drainRequests(); } -// GrpcStreamCallbacks for GrpcMuxDelta -void GrpcMuxDelta::onStreamEstablished() { handleEstablishedStream(); } -void GrpcMuxDelta::onEstablishmentFailure() { handleStreamEstablishmentFailure(); } -void GrpcMuxDelta::onWriteable() { trySendDiscoveryRequests(); } -void GrpcMuxDelta::onDiscoveryResponse( - std::unique_ptr&& message) { - genericHandleResponse(message->type_url(), message.get()); +void GrpcMuxImpl::onStreamEstablished() { + first_stream_request_ = true; + for (const auto& type_url : subscriptions_) { + queueDiscoveryRequest(type_url); + } } -void GrpcMuxDelta::establishGrpcStream() { grpc_stream_.establishNewStream(); } -void GrpcMuxDelta::sendGrpcMessage(void* msg_proto_ptr) { - std::unique_ptr typed_proto( - static_cast(msg_proto_ptr)); - if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { - typed_proto->mutable_node()->MergeFrom(local_info().node()); +void GrpcMuxImpl::onEstablishmentFailure() { + for (const auto& api_state : api_state_) { + for (auto watch : api_state.second.watches_) { + watch->callbacks_.onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); + } } - grpc_stream_.sendMessage(*typed_proto); - set_any_request_sent_yet_in_current_stream(true); } -void GrpcMuxDelta::maybeUpdateQueueSizeStat(uint64_t size) { - grpc_stream_.maybeUpdateQueueSizeStat(size); -} -bool GrpcMuxDelta::grpcStreamAvailable() const { return grpc_stream_.grpcStreamAvailable(); } -bool GrpcMuxDelta::rateLimitAllowsDrain() { return grpc_stream_.checkRateLimitAllowsDrain(); } -GrpcMuxSotw::GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) - : GrpcMuxImpl(std::make_unique(dispatcher), skip_subsequent_node, - local_info), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} - -// GrpcStreamCallbacks for GrpcMuxSotw -void GrpcMuxSotw::onStreamEstablished() { handleEstablishedStream(); } -void GrpcMuxSotw::onEstablishmentFailure() { handleStreamEstablishmentFailure(); } -void GrpcMuxSotw::onWriteable() { trySendDiscoveryRequests(); } -void GrpcMuxSotw::onDiscoveryResponse( - std::unique_ptr&& message) { - genericHandleResponse(message->type_url(), message.get()); +void GrpcMuxImpl::queueDiscoveryRequest(const std::string& queue_item) { + request_queue_.push(queue_item); + drainRequests(); } -void GrpcMuxSotw::establishGrpcStream() { grpc_stream_.establishNewStream(); } - -void GrpcMuxSotw::sendGrpcMessage(void* msg_proto_ptr) { - std::unique_ptr typed_proto( - static_cast(msg_proto_ptr)); - if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { - typed_proto->mutable_node()->MergeFrom(local_info().node()); - } - grpc_stream_.sendMessage(*typed_proto); - set_any_request_sent_yet_in_current_stream(true); +void GrpcMuxImpl::clearRequestQueue() { + grpc_stream_.maybeUpdateQueueSizeStat(0); + request_queue_ = {}; } -void GrpcMuxSotw::maybeUpdateQueueSizeStat(uint64_t size) { - grpc_stream_.maybeUpdateQueueSizeStat(size); +void GrpcMuxImpl::drainRequests() { + while (!request_queue_.empty() && grpc_stream_.checkRateLimitAllowsDrain()) { + // Process the request, if rate limiting is not enabled at all or if it is under rate limit. + sendDiscoveryRequest(request_queue_.front()); + request_queue_.pop(); + } + grpc_stream_.maybeUpdateQueueSizeStat(request_queue_.size()); } -bool GrpcMuxSotw::grpcStreamAvailable() const { return grpc_stream_.grpcStreamAvailable(); } -bool GrpcMuxSotw::rateLimitAllowsDrain() { return grpc_stream_.checkRateLimitAllowsDrain(); } - } // namespace Config } // namespace Envoy diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index ff0d5db75e89..3f55178f4e1e 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -1,185 +1,142 @@ #pragma once -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/common/token_bucket.h" +#include +#include + +#include "envoy/common/time.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" +#include "envoy/grpc/status.h" +#include "envoy/upstream/cluster_manager.h" +#include "common/common/cleanup.h" #include "common/common/logger.h" -#include "common/config/delta_subscription_state.h" #include "common/config/grpc_stream.h" -#include "common/config/pausable_ack_queue.h" -#include "common/config/sotw_subscription_state.h" -#include "common/config/watch_map.h" -#include "common/grpc/common.h" +#include "common/config/utility.h" namespace Envoy { namespace Config { -// Manages subscriptions to one or more type of resource. The logical protocol -// state of those subscription(s) is handled by SubscriptionState. -// This class owns the GrpcStream used to talk to the server, maintains queuing -// logic to properly order the subscription(s)' various messages, and allows -// starting/stopping/pausing of the subscriptions. -class GrpcMuxImpl : public GrpcMux, Logger::Loggable { +/** + * ADS API implementation that fetches via gRPC. + */ +class GrpcMuxImpl : public GrpcMux, + public GrpcStreamCallbacks, + public Logger::Loggable { public: - GrpcMuxImpl(std::unique_ptr subscription_state_factory, - bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info); + GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, + Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node); + ~GrpcMuxImpl() override; - Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, - const std::set& resources, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) override; - void removeWatch(const std::string& type_url, Watch* watch) override; + void start() override; + GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, + GrpcMuxCallbacks& callbacks) override; + // GrpcMux + // TODO(fredlas) PR #8478 will remove this. + bool isDelta() const override { return false; } void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; bool paused(const std::string& type_url) const override; - void start() override; - void disableInitFetchTimeoutTimer() override; - -protected: - // Everything related to GrpcStream must remain abstract. GrpcStream (and the gRPC-using classes - // that underlie it) are templated on protobufs. That means that a single implementation that - // supports different types of protobufs cannot use polymorphism to share code. The workaround: - // the GrpcStream will be owned by a derived class, and all code that would touch grpc_stream_ is - // seen here in the base class as calls to abstract functions, to be provided by those derived - // classes. - virtual void establishGrpcStream() PURE; - // Deletes msg_proto_ptr. - virtual void sendGrpcMessage(void* msg_proto_ptr) PURE; - virtual void maybeUpdateQueueSizeStat(uint64_t size) PURE; - virtual bool grpcStreamAvailable() const PURE; - virtual bool rateLimitAllowsDrain() PURE; - - SubscriptionState& subscriptionStateFor(const std::string& type_url); - WatchMap& watchMapFor(const std::string& type_url); - void handleEstablishedStream(); - void handleStreamEstablishmentFailure(); - void genericHandleResponse(const std::string& type_url, const void* response_proto_ptr); - void trySendDiscoveryRequests(); - bool skip_subsequent_node() const { return skip_subsequent_node_; } - bool any_request_sent_yet_in_current_stream() const { - return any_request_sent_yet_in_current_stream_; - } - void set_any_request_sent_yet_in_current_stream(bool value) { - any_request_sent_yet_in_current_stream_ = value; - } - const LocalInfo::LocalInfo& local_info() const { return local_info_; } -private: - Watch* addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); - - // Updates the list of resource names watched by the given watch. If an added name is new across - // the whole subscription, or if a removed name has no other watch interested in it, then the - // subscription will enqueue and attempt to send an appropriate discovery request. - void updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources); - - // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check - // whether we *want* to send a DeltaDiscoveryRequest). - bool canSendDiscoveryRequest(const std::string& type_url); - - // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or - // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). - // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). - // First, prioritizes ACKs over non-ACK subscription interest updates. - // Then, prioritizes non-ACK updates in the order the various types - // of subscriptions were activated (as tracked by subscription_ordering_). - absl::optional whoWantsToSendDiscoveryRequest(); - - // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All - // of our different resource types' ACKs are mixed together in this queue. See class for - // description of how it interacts with pause() and resume(). - PausableAckQueue pausable_ack_queue_; - - // Makes SubscriptionStates, to be held in the subscriptions_ map. Whether this GrpcMux is doing - // delta or state of the world xDS is determined by which concrete subclass this variable gets. - std::unique_ptr subscription_state_factory_; - - // Map key is type_url. - // Only addWatch() should insert into these maps. - absl::flat_hash_map> subscriptions_; - absl::flat_hash_map> watch_maps_; - - // Determines the order of initial discovery requests. (Assumes that subscriptions are added - // to this GrpcMux in the order of Envoy's dependency ordering). - std::list subscription_ordering_; - - // Whether to enable the optimization of only including the node field in the very first - // discovery request in an xDS gRPC stream (really just one: *not* per-type_url). - const bool skip_subsequent_node_; - - // State to help with skip_subsequent_node's logic. - bool any_request_sent_yet_in_current_stream_{}; + Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + void removeWatch(const std::string&, Watch*) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - // Used to populate the [Delta]DiscoveryRequest's node field. That field is the same across - // all type_urls, and moreover, the 'skip_subsequent_node' logic needs to operate across all - // the type_urls. So, while the SubscriptionStates populate every other field of these messages, - // this one is up to GrpcMux. - const LocalInfo::LocalInfo& local_info_; -}; + void handleDiscoveryResponse(std::unique_ptr&& message); -class GrpcMuxDelta : public GrpcMuxImpl, - public GrpcStreamCallbacks { -public: - GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node); + void sendDiscoveryRequest(const std::string& type_url); - // GrpcStreamCallbacks + // Config::GrpcStreamCallbacks void onStreamEstablished() override; void onEstablishmentFailure() override; + void onDiscoveryResponse(std::unique_ptr&& message) override; void onWriteable() override; - void - onDiscoveryResponse(std::unique_ptr&& message) override; -protected: - void establishGrpcStream() override; - void sendGrpcMessage(void* msg_proto_ptr) override; - void maybeUpdateQueueSizeStat(uint64_t size) override; - bool grpcStreamAvailable() const override; - bool rateLimitAllowsDrain() override; - -private: - GrpcStream - grpc_stream_; -}; - -class GrpcMuxSotw : public GrpcMuxImpl, - public GrpcStreamCallbacks { -public: - GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node); - - // GrpcStreamCallbacks - void onStreamEstablished() override; - void onEstablishmentFailure() override; - void onWriteable() override; - void onDiscoveryResponse(std::unique_ptr&& message) override; GrpcStream& grpcStreamForTest() { return grpc_stream_; } -protected: - void establishGrpcStream() override; - void sendGrpcMessage(void* msg_proto_ptr) override; - void maybeUpdateQueueSizeStat(uint64_t size) override; - bool grpcStreamAvailable() const override; - bool rateLimitAllowsDrain() override; - private: + void drainRequests(); + void setRetryTimer(); + + struct GrpcMuxWatchImpl : public GrpcMuxWatch, RaiiListElement { + GrpcMuxWatchImpl(const std::set& resources, GrpcMuxCallbacks& callbacks, + const std::string& type_url, GrpcMuxImpl& parent) + : RaiiListElement(parent.api_state_[type_url].watches_, this), + resources_(resources), callbacks_(callbacks), type_url_(type_url), parent_(parent), + inserted_(true) {} + ~GrpcMuxWatchImpl() override { + if (inserted_) { + erase(); + if (!resources_.empty()) { + parent_.sendDiscoveryRequest(type_url_); + } + } + } + + void clear() { + inserted_ = false; + cancel(); + } + + std::set resources_; + GrpcMuxCallbacks& callbacks_; + const std::string type_url_; + GrpcMuxImpl& parent_; + + private: + bool inserted_; + }; + + // Per muxed API state. + struct ApiState { + // Watches on the returned resources for the API; + std::list watches_; + // Current DiscoveryRequest for API. + envoy::api::v2::DiscoveryRequest request_; + // Paused via pause()? + bool paused_{}; + // Was a DiscoveryRequest elided during a pause? + bool pending_{}; + // Has this API been tracked in subscriptions_? + bool subscribed_{}; + }; + + // Request queue management logic. + void queueDiscoveryRequest(const std::string& queue_item); + void clearRequestQueue(); + GrpcStream grpc_stream_; + const LocalInfo::LocalInfo& local_info_; + const bool skip_subsequent_node_; + bool first_stream_request_; + std::unordered_map api_state_; + // Envoy's dependency ordering. + std::list subscriptions_; + + // A queue to store requests while rate limited. Note that when requests cannot be sent due to the + // gRPC stream being down, this queue does not store them; rather, they are simply dropped. + // This string is a type URL. + std::queue request_queue_; }; -class NullGrpcMuxImpl : public GrpcMux { +class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks { public: void start() override {} - + GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } + // TODO(fredlas) PR #8478 will remove this. + bool isDelta() const override { return false; } void pause(const std::string&) override {} void resume(const std::string&) override {} bool paused(const std::string&) const override { return false; } @@ -191,7 +148,11 @@ class NullGrpcMuxImpl : public GrpcMux { void removeWatch(const std::string&, Watch*) override { throw EnvoyException("ADS must be configured to support an ADS config source"); } - void disableInitFetchTimeoutTimer() override {} + + void onWriteable() override {} + void onStreamEstablished() override {} + void onEstablishmentFailure() override {} + void onDiscoveryResponse(std::unique_ptr&&) override {} }; } // namespace Config diff --git a/source/common/config/grpc_mux_subscription_impl.cc b/source/common/config/grpc_mux_subscription_impl.cc new file mode 100644 index 000000000000..06d1fd3b562f --- /dev/null +++ b/source/common/config/grpc_mux_subscription_impl.cc @@ -0,0 +1,108 @@ +#include "common/config/grpc_mux_subscription_impl.h" + +#include "common/common/assert.h" +#include "common/common/logger.h" +#include "common/grpc/common.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Config { + +GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, + SubscriptionCallbacks& callbacks, + SubscriptionStats stats, + absl::string_view type_url, + Event::Dispatcher& dispatcher, + std::chrono::milliseconds init_fetch_timeout) + : grpc_mux_(grpc_mux), callbacks_(callbacks), stats_(stats), type_url_(type_url), + dispatcher_(dispatcher), init_fetch_timeout_(init_fetch_timeout) {} + +// Config::Subscription +void GrpcMuxSubscriptionImpl::start(const std::set& resources) { + if (init_fetch_timeout_.count() > 0) { + init_fetch_timeout_timer_ = dispatcher_.createTimer([this]() -> void { + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, + nullptr); + }); + init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); + } + + watch_ = grpc_mux_->subscribe(type_url_, resources, *this); + // The attempt stat here is maintained for the purposes of having consistency between ADS and + // gRPC/filesystem/REST Subscriptions. Since ADS is push based and muxed, the notion of an + // "attempt" for a given xDS API combined by ADS is not really that meaningful. + stats_.update_attempt_.inc(); +} + +void GrpcMuxSubscriptionImpl::updateResourceInterest( + const std::set& update_to_these_names) { + // First destroy the watch, so that this subscribe doesn't send a request for both the + // previously watched resources and the new ones (we may have lost interest in some of the + // previously watched ones). + watch_.reset(); + watch_ = grpc_mux_->subscribe(type_url_, update_to_these_names, *this); + stats_.update_attempt_.inc(); +} + +// Config::GrpcMuxCallbacks +void GrpcMuxSubscriptionImpl::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + disableInitFetchTimeoutTimer(); + // TODO(mattklein123): In the future if we start tracking per-resource versions, we need to + // supply those versions to onConfigUpdate() along with the xDS response ("system") + // version_info. This way, both types of versions can be tracked and exposed for debugging by + // the configuration update targets. + callbacks_.onConfigUpdate(resources, version_info); + stats_.update_success_.inc(); + stats_.update_attempt_.inc(); + stats_.version_.set(HashUtil::xxHash64(version_info)); + ENVOY_LOG(debug, "gRPC config for {} accepted with {} resources with version {}", type_url_, + resources.size(), version_info); +} + +void GrpcMuxSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, + const EnvoyException* e) { + switch (reason) { + case Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure: + stats_.update_failure_.inc(); + ENVOY_LOG(debug, "gRPC update for {} failed", type_url_); + break; + case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: + stats_.init_fetch_timeout_.inc(); + disableInitFetchTimeoutTimer(); + ENVOY_LOG(warn, "gRPC config: initial fetch timed out for {}", type_url_); + break; + case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: + // We expect Envoy exception to be thrown when update is rejected. + ASSERT(e != nullptr); + disableInitFetchTimeoutTimer(); + stats_.update_rejected_.inc(); + ENVOY_LOG(warn, "gRPC config for {} rejected: {}", type_url_, e->what()); + break; + } + + stats_.update_attempt_.inc(); + if (reason == Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure) { + // New gRPC stream will be established and send requests again. + // If init_fetch_timeout is non-zero, server will continue startup after it timeout + return; + } + + callbacks_.onConfigUpdateFailed(reason, e); +} + +std::string GrpcMuxSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { + return callbacks_.resourceName(resource); +} + +void GrpcMuxSubscriptionImpl::disableInitFetchTimeoutTimer() { + if (init_fetch_timeout_timer_) { + init_fetch_timeout_timer_->disableTimer(); + init_fetch_timeout_timer_.reset(); + } +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h new file mode 100644 index 000000000000..299aa4f1480e --- /dev/null +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -0,0 +1,50 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/config/grpc_mux.h" +#include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" + +#include "common/common/logger.h" + +namespace Envoy { +namespace Config { + +/** + * Adapter from typed Subscription to untyped GrpcMux. Also handles per-xDS API stats/logging. + */ +class GrpcMuxSubscriptionImpl : public Subscription, + GrpcMuxCallbacks, + Logger::Loggable { +public: + GrpcMuxSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, SubscriptionCallbacks& callbacks, + SubscriptionStats stats, absl::string_view type_url, + Event::Dispatcher& dispatcher, + std::chrono::milliseconds init_fetch_timeout); + + // Config::Subscription + void start(const std::set& resource_names) override; + void updateResourceInterest(const std::set& update_to_these_names) override; + + // Config::GrpcMuxCallbacks + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override; + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) override; + std::string resourceName(const ProtobufWkt::Any& resource) override; + +private: + void disableInitFetchTimeoutTimer(); + + GrpcMuxSharedPtr grpc_mux_; + SubscriptionCallbacks& callbacks_; + SubscriptionStats stats_; + const std::string type_url_; + GrpcMuxWatchPtr watch_{}; + Event::Dispatcher& dispatcher_; + std::chrono::milliseconds init_fetch_timeout_; + Event::TimerPtr init_fetch_timeout_timer_; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index 328462d65e65..60cf40099a55 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -96,10 +96,15 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, } void maybeUpdateQueueSizeStat(uint64_t size) { - // A change in queue length is not "meaningful" until it has persisted until here. - if (size != last_queue_size_) { + // Although request_queue_.push() happens elsewhere, the only time the queue is non-transiently + // non-empty is when it remains non-empty after a drain attempt. (The push() doesn't matter + // because we always attempt this drain immediately after the push). Basically, a change in + // queue length is not "meaningful" until it has persisted until here. We need the + // if(>0 || used) to keep this stat from being wrongly marked interesting by a pointless set(0) + // and needlessly taking up space. The first time we set(123), used becomes true, and so we will + // subsequently always do the set (including set(0)). + if (size > 0 || control_plane_stats_.pending_requests_.used()) { control_plane_stats_.pending_requests_.set(size); - last_queue_size_ = size; } } @@ -142,7 +147,6 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, TokenBucketPtr limit_request_; const bool rate_limiting_enabled_; Event::TimerPtr drain_request_timer_; - uint64_t last_queue_size_{}; }; } // namespace Config diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 46968aa3d71b..8914519746f2 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -1,68 +1,49 @@ #pragma once +#include "envoy/api/v2/core/base.pb.h" #include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" +#include "envoy/grpc/async_client.h" #include "common/config/grpc_mux_impl.h" +#include "common/config/grpc_mux_subscription_impl.h" #include "common/config/utility.h" namespace Envoy { namespace Config { -// GrpcSubscriptionImpl provides a top-level interface to the Envoy's gRPC communication with -// an xDS server, for use by the various xDS users within Envoy. It is built around a (shared) -// GrpcMuxImpl, and the further machinery underlying that. An xDS user indicates interest in -// various resources via start() and updateResourceInterest(). It receives updates to those -// resources via the SubscriptionCallbacks it provides. Multiple users can each have their own -// Subscription object for the same type_url; GrpcMuxImpl maintains a subscription to the -// union of interested resources, and delivers to the users just the resource updates that they -// are "watching" for. -// -// GrpcSubscriptionImpl and GrpcMuxImpl are both built to provide both regular xDS and ADS, -// distinguished by whether multiple GrpcSubscriptionImpls are sharing a single GrpcMuxImpl. -// (Also distinguished by the gRPC method string, but that's taken care of in SubscriptionFactory). -// -// Why does GrpcSubscriptionImpl itself implement the SubscriptionCallbacks interface? So that it -// can write to SubscriptionStats (which needs to live out here in the GrpcSubscriptionImpl) upon a -// config update. GrpcSubscriptionImpl presents itself to WatchMap as the SubscriptionCallbacks, -// and then, after incrementing stats, passes through to the real callbacks_. -class GrpcSubscriptionImpl : public Subscription, public SubscriptionCallbacks { +class GrpcSubscriptionImpl : public Config::Subscription { public: - // is_aggregated: whether our GrpcMux is also providing ADS to other Subscriptions, or whether - // it's all ours. The practical difference is that we ourselves must call start() on it only if - // we are the sole owner. - GrpcSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, absl::string_view type_url, + GrpcSubscriptionImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, + Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, + const Protobuf::MethodDescriptor& service_method, absl::string_view type_url, SubscriptionCallbacks& callbacks, SubscriptionStats stats, - std::chrono::milliseconds init_fetch_timeout, bool is_aggregated); - ~GrpcSubscriptionImpl() override; - - void pause(); - void resume(); + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + std::chrono::milliseconds init_fetch_timeout, bool skip_subsequent_node) + : callbacks_(callbacks), + grpc_mux_(std::make_shared(local_info, std::move(async_client), + dispatcher, service_method, random, scope, + rate_limit_settings, skip_subsequent_node)), + grpc_mux_subscription_(grpc_mux_, callbacks_, stats, type_url, dispatcher, + init_fetch_timeout) {} // Config::Subscription - void start(const std::set& resource_names) override; - void updateResourceInterest(const std::set& update_to_these_names) override; + void start(const std::set& resource_names) override { + // Subscribe first, so we get failure callbacks if grpc_mux_.start() fails. + grpc_mux_subscription_.start(resource_names); + grpc_mux_->start(); + } - // Config::SubscriptionCallbacks (all pass through to callbacks_!) - void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) override; - void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override; - void onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) override; - std::string resourceName(const ProtobufWkt::Any& resource) override; + void updateResourceInterest(const std::set& update_to_these_names) override { + grpc_mux_subscription_.updateResourceInterest(update_to_these_names); + } - GrpcMuxSharedPtr getGrpcMuxForTest() { return grpc_mux_; } + std::shared_ptr grpcMux() { return grpc_mux_; } private: - GrpcMuxSharedPtr const grpc_mux_; - const std::string type_url_; - SubscriptionCallbacks& callbacks_; - SubscriptionStats stats_; - // NOTE: if another subscription of the same type_url has already been started, the value of - // init_fetch_timeout_ will be ignored in favor of the other subscription's. - const std::chrono::milliseconds init_fetch_timeout_; - Watch* watch_{}; - const bool is_aggregated_; + Config::SubscriptionCallbacks& callbacks_; + std::shared_ptr grpc_mux_; + GrpcMuxSubscriptionImpl grpc_mux_subscription_; }; } // namespace Config diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc new file mode 100644 index 000000000000..d22593d0bf99 --- /dev/null +++ b/source/common/config/new_grpc_mux_impl.cc @@ -0,0 +1,228 @@ +#include "common/config/new_grpc_mux_impl.h" + +#include "common/common/assert.h" +#include "common/common/backoff_strategy.h" +#include "common/common/token_bucket_impl.h" +#include "common/config/utility.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Config { + +NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, + Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info) + : dispatcher_(dispatcher), local_info_(local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + +Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + if (watch == nullptr) { + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } else { + updateWatch(type_url, watch, resources); + return watch; + } +} + +void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { + updateWatch(type_url, watch, {}); + auto entry = subscriptions_.find(type_url); + RELEASE_ASSERT(entry != subscriptions_.end(), + fmt::format("removeWatch() called for non-existent subscription {}.", type_url)); + entry->second->watch_map_.removeWatch(watch); +} + +void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } + +void NewGrpcMuxImpl::resume(const std::string& type_url) { + pausable_ack_queue_.resume(type_url); + trySendDiscoveryRequests(); +} + +bool NewGrpcMuxImpl::paused(const std::string& type_url) const { + return pausable_ack_queue_.paused(type_url); +} + +void NewGrpcMuxImpl::onDiscoveryResponse( + std::unique_ptr&& message) { + ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), + message->system_version_info()); + auto sub = subscriptions_.find(message->type_url()); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, + "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " + "subscription {}.", + message->system_version_info(), message->type_url()); + return; + } + kickOffAck(sub->second->sub_state_.handleResponse(*message)); +} + +void NewGrpcMuxImpl::onStreamEstablished() { + for (auto& sub : subscriptions_) { + sub.second->sub_state_.markStreamFresh(); + } + trySendDiscoveryRequests(); +} + +void NewGrpcMuxImpl::onEstablishmentFailure() { + // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately + // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a + // crash, the iteration needs to dance around a little: collect pointers to all + // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are + // now more SubscriptionStates. + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; + do { + for (auto& sub : subscriptions_) { + all_subscribed[sub.first] = &sub.second->sub_state_; + } + for (auto& sub : all_subscribed) { + if (already_called.insert(sub).second) { // insert succeeded ==> not already called + sub.second->handleEstablishmentFailure(); + } + } + } while (all_subscribed.size() != subscriptions_.size()); +} + +void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } + +void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { + pausable_ack_queue_.push(std::move(ack)); + trySendDiscoveryRequests(); +} + +// TODO(fredlas) to be removed from the GrpcMux interface very soon. +GrpcMuxWatchPtr NewGrpcMuxImpl::subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +void NewGrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } + +Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + auto entry = subscriptions_.find(type_url); + if (entry == subscriptions_.end()) { + // We don't yet have a subscription for type_url! Make one! + addSubscription(type_url, init_fetch_timeout); + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } + + Watch* watch = entry->second->watch_map_.addWatch(callbacks); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. + updateWatch(type_url, watch, resources); + return watch; +} + +// Updates the list of resource names watched by the given watch. If an added name is new across +// the whole subscription, or if a removed name has no other watch interested in it, then the +// subscription will enqueue and attempt to send an appropriate discovery request. +void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources) { + ASSERT(watch != nullptr); + auto sub = subscriptions_.find(type_url); + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Watch of {} has no subscription to update.", type_url)); + auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); + sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); + // Tell the server about our change in interest, if any. + if (sub->second->sub_state_.subscriptionUpdatePending()) { + trySendDiscoveryRequests(); + } +} + +void NewGrpcMuxImpl::addSubscription(const std::string& type_url, + std::chrono::milliseconds init_fetch_timeout) { + subscriptions_.emplace(type_url, std::make_unique(type_url, init_fetch_timeout, + dispatcher_, local_info_)); + subscription_ordering_.emplace_back(type_url); +} + +void NewGrpcMuxImpl::trySendDiscoveryRequests() { + while (true) { + // Do any of our subscriptions even want to send a request? + absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); + if (!maybe_request_type.has_value()) { + break; + } + // If so, which one (by type_url)? + std::string next_request_type_url = maybe_request_type.value(); + // If we don't have a subscription object for this request's type_url, drop the request. + auto sub = subscriptions_.find(next_request_type_url); + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Tried to send discovery request for non-existent subscription {}.", + next_request_type_url)); + + // Try again later if paused/rate limited/stream down. + if (!canSendDiscoveryRequest(next_request_type_url)) { + break; + } + // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. + if (!pausable_ack_queue_.empty()) { + // Because ACKs take precedence over plain requests, if there is anything in the queue, it's + // safe to assume it's of the type_url that we're wanting to send. + grpc_stream_.sendMessage( + sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); + pausable_ack_queue_.pop(); + } else { + grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); + } + } + grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); +} + +// Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check +// whether we *want* to send a DeltaDiscoveryRequest). +bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { + RELEASE_ASSERT( + !pausable_ack_queue_.paused(type_url), + fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " + "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", + type_url)); + + if (!grpc_stream_.grpcStreamAvailable()) { + ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); + return false; + } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { + ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); + return false; + } + return true; +} + +// Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or +// a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). +// Returns the type_url we should send the DeltaDiscoveryRequest for (if any). +// First, prioritizes ACKs over non-ACK subscription interest updates. +// Then, prioritizes non-ACK updates in the order the various types +// of subscriptions were activated. +absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { + // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose + // type_url from pausable_ack_queue_ if possible, before looking at pending updates. + if (!pausable_ack_queue_.empty()) { + return pausable_ack_queue_.front().type_url_; + } + // If we're looking to send multiple non-ACK requests, send them in the order that their + // subscriptions were initiated. + for (const auto& sub_type : subscription_ordering_) { + auto sub = subscriptions_.find(sub_type); + if (sub != subscriptions_.end() && sub->second->sub_state_.subscriptionUpdatePending() && + !pausable_ack_queue_.paused(sub_type)) { + return sub->first; + } + } + return absl::nullopt; +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h new file mode 100644 index 000000000000..4315138b0d44 --- /dev/null +++ b/source/common/config/new_grpc_mux_impl.h @@ -0,0 +1,120 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/token_bucket.h" +#include "envoy/config/grpc_mux.h" +#include "envoy/config/subscription.h" + +#include "common/common/logger.h" +#include "common/config/delta_subscription_state.h" +#include "common/config/grpc_stream.h" +#include "common/config/pausable_ack_queue.h" +#include "common/config/watch_map.h" +#include "common/grpc/common.h" + +namespace Envoy { +namespace Config { + +// Manages subscriptions to one or more type of resource. The logical protocol +// state of those subscription(s) is handled by DeltaSubscriptionState. +// This class owns the GrpcStream used to talk to the server, maintains queuing +// logic to properly order the subscription(s)' various messages, and allows +// starting/stopping/pausing of the subscriptions. +class NewGrpcMuxImpl : public GrpcMux, + public GrpcStreamCallbacks, + Logger::Loggable { +public: + NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info); + + Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + void removeWatch(const std::string& type_url, Watch* watch) override; + + // TODO(fredlas) PR #8478 will remove this. + bool isDelta() const override { return true; } + + void pause(const std::string& type_url) override; + void resume(const std::string& type_url) override; + bool paused(const std::string& type_url) const override; + void + onDiscoveryResponse(std::unique_ptr&& message) override; + + void onStreamEstablished() override; + + void onEstablishmentFailure() override; + + void onWriteable() override; + + void kickOffAck(UpdateAck ack); + + // TODO(fredlas) remove these two from the GrpcMux interface. + GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) override; + void start() override; + +private: + Watch* addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); + + // Updates the list of resource names watched by the given watch. If an added name is new across + // the whole subscription, or if a removed name has no other watch interested in it, then the + // subscription will enqueue and attempt to send an appropriate discovery request. + void updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources); + + void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); + + void trySendDiscoveryRequests(); + + // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check + // whether we *want* to send a DeltaDiscoveryRequest). + bool canSendDiscoveryRequest(const std::string& type_url); + + // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or + // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). + // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). + // First, prioritizes ACKs over non-ACK subscription interest updates. + // Then, prioritizes non-ACK updates in the order the various types + // of subscriptions were activated. + absl::optional whoWantsToSendDiscoveryRequest(); + + Event::Dispatcher& dispatcher_; + const LocalInfo::LocalInfo& local_info_; + + // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All + // of our different resource types' ACKs are mixed together in this queue. See class for + // description of how it interacts with pause() and resume(). + PausableAckQueue pausable_ack_queue_; + + struct SubscriptionStuff { + SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) + : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), + init_fetch_timeout_(init_fetch_timeout) {} + + WatchMap watch_map_; + DeltaSubscriptionState sub_state_; + const std::chrono::milliseconds init_fetch_timeout_; + + SubscriptionStuff(const SubscriptionStuff&) = delete; + SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; + }; + // Map key is type_url. + absl::flat_hash_map> subscriptions_; + + // Determines the order of initial discovery requests. (Assumes that subscriptions are added in + // the order of Envoy's dependency ordering). + std::list subscription_ordering_; + + GrpcStream + grpc_stream_; +}; + +using NewGrpcMuxImplSharedPtr = std::shared_ptr; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/pausable_ack_queue.h b/source/common/config/pausable_ack_queue.h index 875ff3b2f23b..46222a8b2e3c 100644 --- a/source/common/config/pausable_ack_queue.h +++ b/source/common/config/pausable_ack_queue.h @@ -2,13 +2,21 @@ #include -#include "common/config/update_ack.h" +#include "envoy/api/v2/discovery.pb.h" #include "absl/container/flat_hash_map.h" namespace Envoy { namespace Config { +struct UpdateAck { + UpdateAck(absl::string_view nonce, absl::string_view type_url) + : nonce_(nonce), type_url_(type_url) {} + std::string nonce_; + std::string type_url_; + ::google::rpc::Status error_detail_; +}; + // There is a head-of-line blocking issue resulting from the intersection of 1) ADS's need for // subscription request ordering and 2) the ability to "pause" one of the resource types within ADS. // We need a queue that understands ADS's resource type pausing. Specifically, we need front()/pop() diff --git a/source/common/config/sotw_subscription_state.cc b/source/common/config/sotw_subscription_state.cc deleted file mode 100644 index 956716876ac6..000000000000 --- a/source/common/config/sotw_subscription_state.cc +++ /dev/null @@ -1,128 +0,0 @@ -#include "common/config/sotw_subscription_state.h" - -#include "common/common/assert.h" -#include "common/common/hash.h" - -namespace Envoy { -namespace Config { - -SotwSubscriptionState::SotwSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher) - : SubscriptionState(std::move(type_url), callbacks, init_fetch_timeout, dispatcher) {} - -SotwSubscriptionState::~SotwSubscriptionState() = default; - -SotwSubscriptionStateFactory::SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher) - : dispatcher_(dispatcher) {} - -SotwSubscriptionStateFactory::~SotwSubscriptionStateFactory() = default; - -std::unique_ptr -SotwSubscriptionStateFactory::makeSubscriptionState(const std::string& type_url, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - return std::make_unique(type_url, callbacks, init_fetch_timeout, - dispatcher_); -} - -void SotwSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, - const std::set& cur_removed) { - for (const auto& a : cur_added) { - names_tracked_.insert(a); - } - for (const auto& r : cur_removed) { - names_tracked_.erase(r); - } - if (!cur_added.empty() || !cur_removed.empty()) { - update_pending_ = true; - } -} - -// Not having sent any requests yet counts as an "update pending" since you're supposed to resend -// the entirety of your interest at the start of a stream, even if nothing has changed. -bool SotwSubscriptionState::subscriptionUpdatePending() const { return update_pending_; } - -void SotwSubscriptionState::markStreamFresh() { - last_good_version_info_ = absl::nullopt; - last_good_nonce_ = absl::nullopt; - update_pending_ = true; -} - -UpdateAck SotwSubscriptionState::handleResponse(const void* response_proto_ptr) { - auto* response = static_cast(response_proto_ptr); - // We *always* copy the response's nonce into the next request, even if we're going to make that - // request a NACK by setting error_detail. - UpdateAck ack(response->nonce(), type_url()); - try { - handleGoodResponse(*response); - } catch (const EnvoyException& e) { - handleBadResponse(e, ack); - } - return ack; -} - -void SotwSubscriptionState::handleGoodResponse(const envoy::api::v2::DiscoveryResponse& message) { - disableInitFetchTimeoutTimer(); - for (const auto& resource : message.resources()) { - if (resource.type_url() != type_url()) { - throw EnvoyException(fmt::format("type URL {} embedded in an individual Any does not match " - "the message-wide type URL {} in DiscoveryResponse {}", - resource.type_url(), type_url(), message.DebugString())); - } - } - callbacks().onConfigUpdate(message.resources(), message.version_info()); - // Now that we're passed onConfigUpdate() without an exception thrown, we know we're good. - last_good_version_info_ = message.version_info(); - last_good_nonce_ = message.nonce(); - ENVOY_LOG(debug, "Config update for {} accepted with {} resources", type_url(), - message.resources().size()); -} - -void SotwSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAck& ack) { - // Note that error_detail being set is what indicates that a DeltaDiscoveryRequest is a NACK. - ack.error_detail_.set_code(Grpc::Status::WellKnownGrpcStatus::Internal); - ack.error_detail_.set_message(e.what()); - disableInitFetchTimeoutTimer(); - ENVOY_LOG(warn, "gRPC state-of-the-world config for {} rejected: {}", type_url(), e.what()); - callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); -} - -void SotwSubscriptionState::handleEstablishmentFailure() { - callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, - nullptr); -} - -envoy::api::v2::DiscoveryRequest* SotwSubscriptionState::getNextRequestInternal() { - auto* request = new envoy::api::v2::DiscoveryRequest; - request->set_type_url(type_url()); - - std::copy(names_tracked_.begin(), names_tracked_.end(), - Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names())); - if (last_good_version_info_.has_value()) { - request->set_version_info(last_good_version_info_.value()); - } - // Default response_nonce to the last known good one. If we are being called by - // getNextRequestWithAck(), this value will be overwritten. - if (last_good_nonce_.has_value()) { - request->set_response_nonce(last_good_nonce_.value()); - } - - update_pending_ = false; - return request; -} - -void* SotwSubscriptionState::getNextRequestAckless() { return getNextRequestInternal(); } - -void* SotwSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { - envoy::api::v2::DiscoveryRequest* request = getNextRequestInternal(); - request->set_response_nonce(ack.nonce_); - if (ack.error_detail_.code() != Grpc::Status::WellKnownGrpcStatus::Ok) { - // Don't needlessly make the field present-but-empty if status is ok. - request->mutable_error_detail()->CopyFrom(ack.error_detail_); - } - return request; -} - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/sotw_subscription_state.h b/source/common/config/sotw_subscription_state.h deleted file mode 100644 index 1133341e4539..000000000000 --- a/source/common/config/sotw_subscription_state.h +++ /dev/null @@ -1,83 +0,0 @@ -#pragma once - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/grpc/status.h" - -#include "common/common/assert.h" -#include "common/common/hash.h" -#include "common/config/subscription_state.h" - -#include "absl/types/optional.h" - -namespace Envoy { -namespace Config { - -// Tracks the state of a "state-of-the-world" (i.e. not delta) xDS-over-gRPC protocol session. -class SotwSubscriptionState : public SubscriptionState { -public: - // Note that, outside of tests, we expect callbacks to always be a WatchMap. - SotwSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher); - ~SotwSubscriptionState() override; - - // Update which resources we're interested in subscribing to. - void updateSubscriptionInterest(const std::set& cur_added, - const std::set& cur_removed) override; - - // Whether there was a change in our subscription interest we have yet to inform the server of. - bool subscriptionUpdatePending() const override; - - void markStreamFresh() override; - - // message is expected to be a envoy::api::v2::DiscoveryResponse. - UpdateAck handleResponse(const void* response_proto_ptr) override; - - void handleEstablishmentFailure() override; - - // Returns the next gRPC request proto to be sent off to the server, based on this object's - // understanding of the current protocol state, and new resources that Envoy wants to request. - // Returns a new'd pointer, meant to be owned by the caller. - void* getNextRequestAckless() override; - // The WithAck version first calls the Ackless version, then adds in the passed-in ack. - // Returns a new'd pointer, meant to be owned by the caller. - void* getNextRequestWithAck(const UpdateAck& ack) override; - - SotwSubscriptionState(const SotwSubscriptionState&) = delete; - SotwSubscriptionState& operator=(const SotwSubscriptionState&) = delete; - -private: - // Returns a new'd pointer, meant to be owned by the caller. - envoy::api::v2::DiscoveryRequest* getNextRequestInternal(); - - void handleGoodResponse(const envoy::api::v2::DiscoveryResponse& message); - void handleBadResponse(const EnvoyException& e, UpdateAck& ack); - - // The version_info carried by the last accepted DiscoveryResponse. - // Remains empty until one is accepted. - absl::optional last_good_version_info_; - // The nonce carried by the last accepted DiscoveryResponse. - // Remains empty until one is accepted. - // Used when it's time to make a spontaneous (i.e. not primarily meant as an ACK) request. - absl::optional last_good_nonce_; - - // Starts true because we should send a request upon subscription start. - bool update_pending_{true}; - - absl::flat_hash_set names_tracked_; -}; - -class SotwSubscriptionStateFactory : public SubscriptionStateFactory { -public: - SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher); - ~SotwSubscriptionStateFactory() override; - std::unique_ptr - makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) override; - -private: - Event::Dispatcher& dispatcher_; -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index b514868b7b01..9217b612b18d 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -1,9 +1,11 @@ #include "common/config/subscription_factory_impl.h" +#include "common/config/delta_subscription_impl.h" #include "common/config/filesystem_subscription_impl.h" -#include "common/config/grpc_mux_impl.h" +#include "common/config/grpc_mux_subscription_impl.h" #include "common/config/grpc_subscription_impl.h" #include "common/config/http_subscription_impl.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/type_to_endpoint.h" #include "common/config/utility.h" #include "common/protobuf/protobuf.h" @@ -27,8 +29,9 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( switch (config.config_source_specifier_case()) { case envoy::api::v2::core::ConfigSource::kPath: { Utility::checkFilesystemSubscriptionBackingPath(config.path(), api_); - return std::make_unique(dispatcher_, config.path(), callbacks, - stats, validation_visitor_, api_); + result = std::make_unique( + dispatcher_, config.path(), callbacks, stats, validation_visitor_, api_); + break; } case envoy::api::v2::core::ConfigSource::kApiConfigSource: { const envoy::api::v2::core::ApiConfigSource& api_config_source = config.api_config_source(); @@ -40,46 +43,56 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( "Please specify an explicit supported api_type in the following config:\n" + config.DebugString()); case envoy::api::v2::core::ApiConfigSource::REST: - return std::make_unique( + result = std::make_unique( local_info_, cm_, api_config_source.cluster_names()[0], dispatcher_, random_, Utility::apiConfigSourceRefreshDelay(api_config_source), Utility::apiConfigSourceRequestTimeout(api_config_source), restMethod(type_url), callbacks, stats, Utility::configSourceInitialFetchTimeout(config), validation_visitor_); + break; case envoy::api::v2::core::ApiConfigSource::GRPC: - return std::make_unique( - std::make_shared(Utility::factoryForGrpcApiConfigSource( - cm_.grpcAsyncClientManager(), api_config_source, scope) - ->create(), - dispatcher_, sotwGrpcMethod(type_url), random_, scope, - Utility::parseRateLimitSettings(api_config_source), - local_info_, - api_config_source.set_node_on_first_message_only()), - type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), - /*is_aggregated=*/false); - case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: - return std::make_unique( - std::make_shared(Utility::factoryForGrpcApiConfigSource( - cm_.grpcAsyncClientManager(), api_config_source, scope) - ->create(), - dispatcher_, deltaGrpcMethod(type_url), random_, scope, - Utility::parseRateLimitSettings(api_config_source), - local_info_, - api_config_source.set_node_on_first_message_only()), - type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), - /*is_aggregated=*/false); + result = std::make_unique( + local_info_, + Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), + api_config_source, scope) + ->create(), + dispatcher_, random_, sotwGrpcMethod(type_url), type_url, callbacks, stats, scope, + Utility::parseRateLimitSettings(api_config_source), + Utility::configSourceInitialFetchTimeout(config), + api_config_source.set_node_on_first_message_only()); + break; + case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: { + Utility::checkApiConfigSourceSubscriptionBackingCluster(cm_.clusters(), api_config_source); + result = std::make_unique( + std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), + api_config_source, scope) + ->create(), + dispatcher_, deltaGrpcMethod(type_url), random_, scope, + Utility::parseRateLimitSettings(api_config_source), local_info_), + type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), false); + break; + } default: NOT_REACHED_GCOVR_EXCL_LINE; } + break; } case envoy::api::v2::core::ConfigSource::kAds: { - return std::make_unique(cm_.adsMux(), type_url, callbacks, stats, - Utility::configSourceInitialFetchTimeout(config), - /*is_aggregated=*/true); + if (cm_.adsMux()->isDelta()) { + result = std::make_unique( + cm_.adsMux(), type_url, callbacks, stats, + Utility::configSourceInitialFetchTimeout(config), true); + } else { + result = std::make_unique( + cm_.adsMux(), callbacks, stats, type_url, dispatcher_, + Utility::configSourceInitialFetchTimeout(config)); + } + break; } default: throw EnvoyException("Missing config source specifier in envoy::api::v2::core::ConfigSource"); } - NOT_REACHED_GCOVR_EXCL_LINE; + return result; } } // namespace Config diff --git a/source/common/config/subscription_state.cc b/source/common/config/subscription_state.cc deleted file mode 100644 index bfac3bc4bb14..000000000000 --- a/source/common/config/subscription_state.cc +++ /dev/null @@ -1,34 +0,0 @@ -#include "common/config/subscription_state.h" - -#include -#include - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/common/pure.h" -#include "envoy/config/subscription.h" - -namespace Envoy { -namespace Config { - -SubscriptionState::SubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher) - : type_url_(std::move(type_url)), callbacks_(callbacks) { - if (init_fetch_timeout.count() > 0 && !init_fetch_timeout_timer_) { - init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { - ENVOY_LOG(warn, "config: initial fetch timed out for {}", type_url_); - callbacks_.onConfigUpdateFailed(ConfigUpdateFailureReason::FetchTimedout, nullptr); - }); - init_fetch_timeout_timer_->enableTimer(init_fetch_timeout); - } -} - -void SubscriptionState::disableInitFetchTimeoutTimer() { - if (init_fetch_timeout_timer_) { - init_fetch_timeout_timer_->disableTimer(); - init_fetch_timeout_timer_.reset(); - } -} - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/subscription_state.h b/source/common/config/subscription_state.h deleted file mode 100644 index 85bd56222def..000000000000 --- a/source/common/config/subscription_state.h +++ /dev/null @@ -1,79 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/common/pure.h" -#include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" - -#include "common/config/update_ack.h" -#include "common/protobuf/protobuf.h" - -#include "absl/strings/string_view.h" - -namespace Envoy { -namespace Config { - -// Tracks the protocol state of an individual ongoing xDS-over-gRPC session, for a single type_url. -// There can be multiple SubscriptionStates active, one per type_url. They will all be -// blissfully unaware of each other's existence, even when their messages are being multiplexed -// together by ADS. -// This is the abstract parent class for both the delta and state-of-the-world xDS variants. -class SubscriptionState : public Logger::Loggable { -public: - // Note that, outside of tests, we expect callbacks to always be a WatchMap. - SubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher); - virtual ~SubscriptionState() = default; - - // Update which resources we're interested in subscribing to. - virtual void updateSubscriptionInterest(const std::set& cur_added, - const std::set& cur_removed) PURE; - - // Whether there was a change in our subscription interest we have yet to inform the server of. - virtual bool subscriptionUpdatePending() const PURE; - - virtual void markStreamFresh() PURE; - - // Implementations expect either a DeltaDiscoveryResponse or DiscoveryResponse. The caller is - // expected to know which it should be providing. - virtual UpdateAck handleResponse(const void* response_proto_ptr) PURE; - - virtual void handleEstablishmentFailure() PURE; - - // Returns the next gRPC request proto to be sent off to the server, based on this object's - // understanding of the current protocol state, and new resources that Envoy wants to request. - // Returns a new'd pointer, meant to be owned by the caller, who is expected to know what type the - // pointer actually is. - virtual void* getNextRequestAckless() PURE; - // The WithAck version first calls the Ackless version, then adds in the passed-in ack. - // Returns a new'd pointer, meant to be owned by the caller, who is expected to know what type the - // pointer actually is. - virtual void* getNextRequestWithAck(const UpdateAck& ack) PURE; - - void disableInitFetchTimeoutTimer(); - -protected: - std::string type_url() const { return type_url_; } - SubscriptionCallbacks& callbacks() const { return callbacks_; } - -private: - const std::string type_url_; - // callbacks_ is expected (outside of tests) to be a WatchMap. - SubscriptionCallbacks& callbacks_; - Event::TimerPtr init_fetch_timeout_timer_; -}; - -class SubscriptionStateFactory { -public: - virtual ~SubscriptionStateFactory() = default; - // Note that, outside of tests, we expect callbacks to always be a WatchMap. - virtual std::unique_ptr - makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) PURE; -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/update_ack.h b/source/common/config/update_ack.h deleted file mode 100644 index 193e86e436a0..000000000000 --- a/source/common/config/update_ack.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include - -#include "envoy/api/v2/discovery.pb.h" - -#include "absl/strings/string_view.h" - -namespace Envoy { -namespace Config { - -struct UpdateAck { - UpdateAck(absl::string_view nonce, absl::string_view type_url) - : nonce_(nonce), type_url_(type_url) {} - std::string nonce_; - std::string type_url_; - ::google::rpc::Status error_detail_; -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index e98c8af17122..9df852c7f712 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -33,7 +33,8 @@ struct Watch { }; // NOTE: Users are responsible for eventually calling removeWatch() on the Watch* returned -// by addWatch(). However, we don't expect there to be new users of this class. +// by addWatch(). We don't expect there to be new users of this class beyond +// NewGrpcMuxImpl and DeltaSubscriptionImpl (TODO(fredlas) to be renamed). // // Manages "watches" of xDS resources. Several xDS callers might ask for a subscription to the same // resource name "X". The xDS machinery must return to each their very own subscription to X. diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 9694c7c9d654..da694ab7ddaf 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -18,6 +18,7 @@ #include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/utility.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/resources.h" #include "common/config/utility.h" #include "common/grpc/async_client_manager_impl.h" @@ -219,6 +220,8 @@ ClusterManagerImpl::ClusterManagerImpl( } } + const auto& dyn_resources = bootstrap.dynamic_resources(); + // Cluster loading happens in two phases: first all the primary clusters are loaded, and then all // the secondary clusters are loaded. As it currently stands all non-EDS clusters and EDS which // load endpoint definition from file are primary and @@ -240,26 +243,33 @@ ClusterManagerImpl::ClusterManagerImpl( // This is the only point where distinction between delta ADS and state-of-the-world ADS is made. // After here, we just have a GrpcMux interface held in ads_mux_, which hides // whether the backing implementation is delta or SotW. - if (bootstrap.dynamic_resources().has_ads_config()) { - auto& ads_config = bootstrap.dynamic_resources().ads_config(); - if (ads_config.api_type() == envoy::api::v2::core::ApiConfigSource::DELTA_GRPC) { - ads_mux_ = std::make_shared( - Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, ads_config, stats) + if (dyn_resources.has_ads_config()) { + if (dyn_resources.ads_config().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC) { + auto& api_config_source = dyn_resources.has_ads_config() + ? dyn_resources.ads_config() + : dyn_resources.cds_config().api_config_source(); + ads_mux_ = std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, api_config_source, + stats) ->create(), main_thread_dispatcher, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.DeltaAggregatedResources"), - random_, stats_, Envoy::Config::Utility::parseRateLimitSettings(ads_config), local_info, - ads_config.set_node_on_first_message_only()); + random_, stats_, + Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), local_info); } else { - ads_mux_ = std::make_shared( - Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, ads_config, stats) + ads_mux_ = std::make_shared( + local_info, + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, + dyn_resources.ads_config(), stats) ->create(), main_thread_dispatcher, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, Envoy::Config::Utility::parseRateLimitSettings(ads_config), local_info, - ads_config.set_node_on_first_message_only()); + random_, stats_, + Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), + bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); } } else { ads_mux_ = std::make_unique(); @@ -296,8 +306,8 @@ ClusterManagerImpl::ClusterManagerImpl( }); // We can now potentially create the CDS API once the backing cluster exists. - if (bootstrap.dynamic_resources().has_cds_config()) { - cds_api_ = factory_.createCds(bootstrap.dynamic_resources().cds_config(), *this); + if (dyn_resources.has_cds_config()) { + cds_api_ = factory_.createCds(dyn_resources.cds_config(), *this); init_helper_.setCds(cds_api_.get()); } else { init_helper_.setCds(nullptr); diff --git a/test/common/config/BUILD b/test/common/config/BUILD index f65907bd3eaf..7367f62e35bb 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -15,7 +15,7 @@ envoy_cc_test( srcs = ["delta_subscription_impl_test.cc"], deps = [ ":delta_subscription_test_harness", - "//source/common/config:grpc_subscription_lib", + "//source/common/config:delta_subscription_lib", "//source/common/stats:isolated_store_lib", "//test/mocks:common_lib", "//test/mocks/config:config_mocks", @@ -31,7 +31,7 @@ envoy_cc_test( name = "delta_subscription_state_test", srcs = ["delta_subscription_state_test.cc"], deps = [ - "//source/common/config:delta_subscription_state_lib", + "//source/common/config:delta_subscription_lib", "//source/common/stats:isolated_store_lib", "//test/mocks:common_lib", "//test/mocks/config:config_mocks", @@ -76,7 +76,28 @@ envoy_cc_test( srcs = ["grpc_mux_impl_test.cc"], deps = [ "//source/common/config:grpc_mux_lib", - "//source/common/config:grpc_subscription_lib", + "//source/common/config:protobuf_link_hacks", + "//source/common/config:resources_lib", + "//source/common/protobuf", + "//source/common/stats:isolated_store_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:pkg_cc_proto", + ], +) + +envoy_cc_test( + name = "new_grpc_mux_impl_test", + srcs = ["new_grpc_mux_impl_test.cc"], + deps = [ + "//source/common/config:new_grpc_mux_lib", "//source/common/config:protobuf_link_hacks", "//source/common/config:resources_lib", "//source/common/protobuf", @@ -139,7 +160,7 @@ envoy_cc_test_library( hdrs = ["delta_subscription_test_harness.h"], deps = [ ":subscription_test_harness", - "//source/common/config:grpc_subscription_lib", + "//source/common/config:delta_subscription_lib", "//source/common/grpc:common_lib", "//test/mocks/config:config_mocks", "//test/mocks/event:event_mocks", @@ -178,22 +199,6 @@ envoy_cc_test_library( ], ) -envoy_cc_test( - name = "sotw_subscription_state_test", - srcs = ["sotw_subscription_state_test.cc"], - deps = [ - "//source/common/config:sotw_subscription_state_lib", - "//source/common/stats:isolated_store_lib", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/test_common:logging_lib", - ], -) - envoy_cc_test( name = "subscription_factory_impl_test", srcs = ["subscription_factory_impl_test.cc"], diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 65777ec1217f..e6b783b6583f 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -11,7 +11,7 @@ class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public te DeltaSubscriptionImplTest() = default; // We need to destroy the subscription before the test's destruction, because the subscription's - // destructor removes its watch from the GrpcMuxDelta, and that removal process involves + // destructor removes its watch from the NewGrpcMuxImpl, and that removal process involves // some things held by the test fixture. void TearDown() override { doSubscriptionTearDown(); } }; @@ -71,8 +71,8 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - auto shared_mux = subscription_->getGrpcMuxForTest(); - static_cast(shared_mux.get())->onDiscoveryResponse(std::move(message)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(message)); } // The server gives us our first version of resource name2. // subscription_ now wants to ACK name1 and then name2 (but can't due to pause). @@ -85,8 +85,8 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - auto shared_mux = subscription_->getGrpcMuxForTest(); - static_cast(shared_mux.get())->onDiscoveryResponse(std::move(message)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(message)); } // The server gives us an updated version of resource name1. // subscription_ now wants to ACK name1A, then name2, then name1B (but can't due to pause). @@ -99,8 +99,8 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - auto shared_mux = subscription_->getGrpcMuxForTest(); - static_cast(shared_mux.get())->onDiscoveryResponse(std::move(message)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(message)); } // All ACK sendMessage()s will happen upon calling resume(). EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)) @@ -135,11 +135,11 @@ TEST(DeltaSubscriptionImplFixturelessTest, NoGrpcStream) { const Protobuf::MethodDescriptor* method_descriptor = Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints"); - std::shared_ptr xds_context = std::make_shared( + std::shared_ptr xds_context = std::make_shared( std::unique_ptr(async_client), dispatcher, *method_descriptor, random, - stats_store, rate_limit_settings, local_info, true); + stats_store, rate_limit_settings, local_info); - auto subscription = std::make_unique( + std::unique_ptr subscription = std::make_unique( xds_context, Config::TypeUrl::get().ClusterLoadAssignment, callbacks, stats, std::chrono::milliseconds(12345), false); diff --git a/test/common/config/delta_subscription_state_test.cc b/test/common/config/delta_subscription_state_test.cc index bdcb52995ab5..f98dfe5aa05e 100644 --- a/test/common/config/delta_subscription_state_test.cc +++ b/test/common/config/delta_subscription_state_test.cc @@ -22,18 +22,13 @@ const char TypeUrl[] = "type.googleapis.com/envoy.api.v2.Cluster"; class DeltaSubscriptionStateTest : public testing::Test { protected: DeltaSubscriptionStateTest() - : state_(TypeUrl, callbacks_, std::chrono::milliseconds(0U), dispatcher_) { + : state_(TypeUrl, callbacks_, local_info_, std::chrono::milliseconds(0U), dispatcher_) { state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name1", "name2", "name3")); } - std::unique_ptr getNextDeltaDiscoveryRequestAckless() { - auto* ptr = static_cast(state_.getNextRequestAckless()); - return std::unique_ptr(ptr); - } - UpdateAck deliverDiscoveryResponse( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, @@ -47,7 +42,7 @@ class DeltaSubscriptionStateTest : public testing::Test { message.set_nonce(nonce.value()); } EXPECT_CALL(callbacks_, onConfigUpdate(_, _, _)).Times(expect_config_update_call ? 1 : 0); - return state_.handleResponse(&message); + return state_.handleResponse(message); } UpdateAck deliverBadDiscoveryResponse( @@ -60,10 +55,11 @@ class DeltaSubscriptionStateTest : public testing::Test { message.set_system_version_info(version_info); message.set_nonce(nonce); EXPECT_CALL(callbacks_, onConfigUpdate(_, _, _)).WillOnce(Throw(EnvoyException("oh no"))); - return state_.handleResponse(&message); + return state_.handleResponse(message); } NiceMock> callbacks_; + NiceMock local_info_; NiceMock dispatcher_; // We start out interested in three resources: name1, name2, and name3. DeltaSubscriptionState state_; @@ -84,15 +80,15 @@ populateRepeatedResource(std::vector> items) TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribe) { { state_.updateSubscriptionInterest({"name4"}, {"name1"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name1")); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1")); } { state_.updateSubscriptionInterest({"name1"}, {"name3", "name4"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name1")); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name3", "name4")); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name1")); + EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name3", "name4")); } } @@ -108,9 +104,9 @@ TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribe) { TEST_F(DeltaSubscriptionStateTest, RemoveThenAdd) { state_.updateSubscriptionInterest({}, {"name3"}); state_.updateSubscriptionInterest({"name3"}, {}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name3")); - EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name3")); + EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); } // Due to how our implementation provides the required behavior tested in RemoveThenAdd, the @@ -124,9 +120,9 @@ TEST_F(DeltaSubscriptionStateTest, RemoveThenAdd) { TEST_F(DeltaSubscriptionStateTest, AddThenRemove) { state_.updateSubscriptionInterest({"name4"}, {}); state_.updateSubscriptionInterest({}, {"name4"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_TRUE(cur_request->resource_names_subscribe().empty()); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name4")); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_TRUE(cur_request.resource_names_subscribe().empty()); + EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name4")); } // add/remove/add == add. @@ -134,9 +130,9 @@ TEST_F(DeltaSubscriptionStateTest, AddRemoveAdd) { state_.updateSubscriptionInterest({"name4"}, {}); state_.updateSubscriptionInterest({}, {"name4"}); state_.updateSubscriptionInterest({"name4"}, {}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); } // remove/add/remove == remove. @@ -144,9 +140,9 @@ TEST_F(DeltaSubscriptionStateTest, RemoveAddRemove) { state_.updateSubscriptionInterest({}, {"name3"}); state_.updateSubscriptionInterest({"name3"}, {}); state_.updateSubscriptionInterest({}, {"name3"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_TRUE(cur_request->resource_names_subscribe().empty()); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name3")); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_TRUE(cur_request.resource_names_subscribe().empty()); + EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name3")); } // Starts with 1,2,3. 4 is added/removed/added. In those same updates, 1,2,3 are @@ -155,18 +151,18 @@ TEST_F(DeltaSubscriptionStateTest, BothAddAndRemove) { state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {"name4"}); state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2", "name3")); } TEST_F(DeltaSubscriptionStateTest, CumulativeUpdates) { state_.updateSubscriptionInterest({"name4"}, {}); state_.updateSubscriptionInterest({"name5"}, {}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4", "name5")); - EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4", "name5")); + EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); } // Verifies that a sequence of good and bad responses from the server all get the appropriate @@ -218,11 +214,11 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); deliverDiscoveryResponse(add1_2, {}, "debugversion1"); state_.markStreamFresh(); // simulate a stream reconnection - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_EQ("version1A", cur_request->initial_resource_versions().at("name1")); - EXPECT_EQ("version2A", cur_request->initial_resource_versions().at("name2")); - EXPECT_EQ(cur_request->initial_resource_versions().end(), - cur_request->initial_resource_versions().find("name3")); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); + EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); + EXPECT_EQ(cur_request.initial_resource_versions().end(), + cur_request.initial_resource_versions().find("name3")); } { @@ -233,11 +229,11 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { *remove2.Add() = "name2"; deliverDiscoveryResponse(add1_3, remove2, "debugversion2"); state_.markStreamFresh(); // simulate a stream reconnection - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_EQ("version1B", cur_request->initial_resource_versions().at("name1")); - EXPECT_EQ(cur_request->initial_resource_versions().end(), - cur_request->initial_resource_versions().find("name2")); - EXPECT_EQ("version3A", cur_request->initial_resource_versions().at("name3")); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_EQ("version1B", cur_request.initial_resource_versions().at("name1")); + EXPECT_EQ(cur_request.initial_resource_versions().end(), + cur_request.initial_resource_versions().find("name2")); + EXPECT_EQ("version3A", cur_request.initial_resource_versions().at("name3")); } { @@ -247,17 +243,17 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { *remove1_3.Add() = "name3"; deliverDiscoveryResponse({}, remove1_3, "debugversion3"); state_.markStreamFresh(); // simulate a stream reconnection - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_TRUE(cur_request->initial_resource_versions().empty()); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_TRUE(cur_request.initial_resource_versions().empty()); } { // ...but our own map should remember our interest. In particular, losing interest in a // resource should cause its name to appear in the next request's resource_names_unsubscribe. state_.updateSubscriptionInterest({"name4"}, {"name1", "name2"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2")); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2")); } } @@ -276,16 +272,16 @@ TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribeAfterReconnect) { state_.updateSubscriptionInterest({"name4"}, {"name1"}); state_.markStreamFresh(); // simulate a stream reconnection - auto cur_request = getNextDeltaDiscoveryRequestAckless(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); // Regarding the resource_names_subscribe field: // name1: do not include: we lost interest. // name2: yes do include: we're interested and we have a version of it. // name3: yes do include: even though we don't have a version of it, we are interested. // name4: yes do include: we are newly interested. (If this wasn't a stream reconnect, only name4 // would belong in this subscribe field). - EXPECT_THAT(cur_request->resource_names_subscribe(), + EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name2", "name3", "name4")); - EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); + EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); } // initial_resource_versions should not be present on messages after the first in a stream. @@ -297,10 +293,10 @@ TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { {{"name1", "version1A"}, {"name2", "version2A"}, {"name3", "version3A"}}); deliverDiscoveryResponse(add_all, {}, "debugversion1"); state_.markStreamFresh(); // simulate a stream reconnection - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_EQ("version1A", cur_request->initial_resource_versions().at("name1")); - EXPECT_EQ("version2A", cur_request->initial_resource_versions().at("name2")); - EXPECT_EQ("version3A", cur_request->initial_resource_versions().at("name3")); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); + EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); + EXPECT_EQ("version3A", cur_request.initial_resource_versions().at("name3")); } // Then, after updating the resources but not reconnecting the stream, verify that initial // versions are not sent. @@ -313,8 +309,8 @@ TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { {"name3", "version3B"}, {"name4", "version4A"}}); deliverDiscoveryResponse(add_all, {}, "debugversion2"); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_TRUE(cur_request->initial_resource_versions().empty()); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + EXPECT_TRUE(cur_request.initial_resource_versions().empty()); } } @@ -363,15 +359,6 @@ TEST_F(DeltaSubscriptionStateTest, AddedAndRemoved) { ack.error_detail_.message()); } -TEST_F(DeltaSubscriptionStateTest, handleEstablishmentFailure) { - // Although establishment failure is not supposed to cause an onConfigUpdateFailed() on the - // ultimate actual subscription callbacks, DeltaSubscriptionState's callbacks are actually - // the WatchMap, which then calls GrpcSubscriptionImpl(s). It is the GrpcSubscriptionImpl - // that will decline to pass on an onConfigUpdateFailed(ConnectionFailure). - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)); - state_.handleEstablishmentFailure(); -} - } // namespace } // namespace Config } // namespace Envoy diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 0d2219390582..bf35490d4b90 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -2,7 +2,7 @@ #include -#include "common/config/grpc_subscription_impl.h" +#include "common/config/delta_subscription_impl.h" #include "common/grpc/common.h" #include "test/common/config/subscription_test_harness.h" @@ -34,11 +34,11 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { node_.set_id("fo0"); EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)); - grpc_mux_ = std::make_shared( + xds_context_ = std::make_shared( std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, - random_, stats_store_, rate_limit_settings_, local_info_, false); - subscription_ = std::make_unique( - grpc_mux_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, + random_, stats_store_, rate_limit_settings_, local_info_); + subscription_ = std::make_unique( + xds_context_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, init_fetch_timeout, false); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); } @@ -148,8 +148,8 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); expectSendMessage({}, {}, Grpc::Status::WellKnownGrpcStatus::Internal, "bad config", {}); } - auto shared_mux = subscription_->getGrpcMuxForTest(); - static_cast(shared_mux.get())->onDiscoveryResponse(std::move(response)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } @@ -189,8 +189,8 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { NiceMock random_; NiceMock local_info_; Grpc::MockAsyncStream async_stream_; - std::shared_ptr grpc_mux_; - std::unique_ptr subscription_; + std::shared_ptr xds_context_; + std::unique_ptr subscription_; std::string last_response_nonce_; std::set last_cluster_names_; Envoy::Config::RateLimitSettings rate_limit_settings_; diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 6ea4d7602d1b..c9fb873d6158 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -48,19 +48,19 @@ class GrpcMuxImplTestBase : public testing::Test { stats_.gauge("control_plane.connected_state", Stats::Gauge::ImportMode::NeverImport)) {} void setup() { - grpc_mux_ = std::make_unique( - std::unique_ptr(async_client_), dispatcher_, + grpc_mux_ = std::make_unique( + local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, rate_limit_settings_, local_info_, true); + random_, stats_, rate_limit_settings_, true); } void setup(const RateLimitSettings& custom_rate_limit_settings) { - grpc_mux_ = std::make_unique( - std::unique_ptr(async_client_), dispatcher_, + grpc_mux_ = std::make_unique( + local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, custom_rate_limit_settings, local_info_, true); + random_, stats_, custom_rate_limit_settings, true); } void expectSendMessage(const std::string& type_url, @@ -85,49 +85,15 @@ class GrpcMuxImplTestBase : public testing::Test { error_detail->set_code(error_code); error_detail->set_message(error_message); } - EXPECT_CALL( - async_stream_, - sendMessageRaw_(Grpc::ProtoBufferEqIgnoreRepeatedFieldOrdering(expected_request), false)); - } - - // These tests were written around GrpcMuxWatch, an RAII type returned by the old subscribe(). - // To preserve these tests for the new code, we need an RAII watch handler. That is - // GrpcSubscriptionImpl, but to keep things simple, we'll fake it. (What we really care about - // is the destructor, which is identical to the real one). - class FakeGrpcSubscription { - public: - FakeGrpcSubscription(GrpcMux* grpc_mux, std::string type_url, Watch* watch) - : grpc_mux_(grpc_mux), type_url_(std::move(type_url)), watch_(watch) {} - ~FakeGrpcSubscription() { grpc_mux_->removeWatch(type_url_, watch_); } - - private: - GrpcMux* const grpc_mux_; - std::string type_url_; - Watch* const watch_; - }; - - FakeGrpcSubscription makeWatch(const std::string& type_url, - const std::set& resources) { - return FakeGrpcSubscription(grpc_mux_.get(), type_url, - grpc_mux_->addOrUpdateWatch(type_url, nullptr, resources, - callbacks_, - std::chrono::milliseconds(0))); - } - - FakeGrpcSubscription - makeWatch(const std::string& type_url, const std::set& resources, - NiceMock>& callbacks) { - return FakeGrpcSubscription(grpc_mux_.get(), type_url, - grpc_mux_->addOrUpdateWatch(type_url, nullptr, resources, callbacks, - std::chrono::milliseconds(0))); + EXPECT_CALL(async_stream_, sendMessageRaw_(Grpc::ProtoBufferEq(expected_request), false)); } NiceMock dispatcher_; NiceMock random_; Grpc::MockAsyncClient* async_client_; Grpc::MockAsyncStream async_stream_; - std::unique_ptr grpc_mux_; - NiceMock> callbacks_; + std::unique_ptr grpc_mux_; + NiceMock callbacks_; NiceMock local_info_; Stats::IsolatedStoreImpl stats_; Envoy::Config::RateLimitSettings rate_limit_settings_; @@ -139,25 +105,33 @@ class GrpcMuxImplTest : public GrpcMuxImplTestBase { Event::SimulatedTimeSystem time_system_; }; -// Validate behavior when multiple type URL watches are maintained, watches are created/destroyed. +// TODO(fredlas) #8478 will delete this. +TEST_F(GrpcMuxImplTest, JustForCoverageTodoDelete) { + setup(); + NullGrpcMuxImpl fake; + EXPECT_FALSE(grpc_mux_->isDelta()); + EXPECT_FALSE(fake.isDelta()); +} + +// Validate behavior when multiple type URL watches are maintained, watches are created/destroyed +// (via RAII). TEST_F(GrpcMuxImplTest, MultipleTypeUrlStreams) { setup(); InSequence s; - - FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); - FakeGrpcSubscription bar_sub = makeWatch("type_url_bar", {}); + auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); + auto bar_sub = grpc_mux_->subscribe("bar", {}, callbacks_); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("type_url_foo", {"x", "y"}, "", true); - expectSendMessage("type_url_bar", {}, ""); + expectSendMessage("foo", {"x", "y"}, "", true); + expectSendMessage("bar", {}, ""); grpc_mux_->start(); EXPECT_EQ(1, control_plane_connected_state_.value()); - expectSendMessage("type_url_bar", {"z"}, ""); - FakeGrpcSubscription bar_z_sub = makeWatch("type_url_bar", {"z"}); - expectSendMessage("type_url_bar", {"zz", "z"}, ""); - FakeGrpcSubscription bar_zz_sub = makeWatch("type_url_bar", {"zz"}); - expectSendMessage("type_url_bar", {"z"}, ""); - expectSendMessage("type_url_bar", {}, ""); - expectSendMessage("type_url_foo", {}, ""); + expectSendMessage("bar", {"z"}, ""); + auto bar_z_sub = grpc_mux_->subscribe("bar", {"z"}, callbacks_); + expectSendMessage("bar", {"zz", "z"}, ""); + auto bar_zz_sub = grpc_mux_->subscribe("bar", {"zz"}, callbacks_); + expectSendMessage("bar", {"z"}, ""); + expectSendMessage("bar", {}, ""); + expectSendMessage("foo", {}, ""); } // Validate behavior when multiple type URL watches are maintained and the stream is reset. @@ -174,13 +148,13 @@ TEST_F(GrpcMuxImplTest, ResetStream) { })); setup(); - FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); - FakeGrpcSubscription bar_sub = makeWatch("type_url_bar", {}); - FakeGrpcSubscription baz_sub = makeWatch("type_url_baz", {"z"}); + auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); + auto bar_sub = grpc_mux_->subscribe("bar", {}, callbacks_); + auto baz_sub = grpc_mux_->subscribe("baz", {"z"}, callbacks_); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("type_url_foo", {"x", "y"}, "", true); - expectSendMessage("type_url_bar", {}, ""); - expectSendMessage("type_url_baz", {"z"}, ""); + expectSendMessage("foo", {"x", "y"}, "", true); + expectSendMessage("bar", {}, ""); + expectSendMessage("baz", {"z"}, ""); grpc_mux_->start(); EXPECT_CALL(callbacks_, @@ -192,122 +166,125 @@ TEST_F(GrpcMuxImplTest, ResetStream) { grpc_mux_->grpcStreamForTest().onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Canceled, ""); EXPECT_EQ(0, control_plane_connected_state_.value()); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("type_url_foo", {"x", "y"}, "", true); - expectSendMessage("type_url_bar", {}, ""); - expectSendMessage("type_url_baz", {"z"}, ""); + expectSendMessage("foo", {"x", "y"}, "", true); + expectSendMessage("bar", {}, ""); + expectSendMessage("baz", {"z"}, ""); timer_cb(); - expectSendMessage("type_url_baz", {}, ""); - expectSendMessage("type_url_foo", {}, ""); + expectSendMessage("baz", {}, ""); + expectSendMessage("foo", {}, ""); } // Validate pause-resume behavior. TEST_F(GrpcMuxImplTest, PauseResume) { setup(); InSequence s; - FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); - grpc_mux_->pause("type_url_foo"); + auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); + grpc_mux_->pause("foo"); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); grpc_mux_->start(); - expectSendMessage("type_url_foo", {"x", "y"}, "", true); - grpc_mux_->resume("type_url_foo"); - grpc_mux_->pause("type_url_bar"); - expectSendMessage("type_url_foo", {"z", "x", "y"}, ""); - FakeGrpcSubscription foo_z_sub = makeWatch("type_url_foo", {"z"}); - grpc_mux_->resume("type_url_bar"); - grpc_mux_->pause("type_url_foo"); - FakeGrpcSubscription foo_zz_sub = makeWatch("type_url_foo", {"zz"}); - expectSendMessage("type_url_foo", {"zz", "z", "x", "y"}, ""); - grpc_mux_->resume("type_url_foo"); - grpc_mux_->pause("type_url_foo"); + expectSendMessage("foo", {"x", "y"}, "", true); + grpc_mux_->resume("foo"); + grpc_mux_->pause("bar"); + expectSendMessage("foo", {"z", "x", "y"}, ""); + auto foo_z_sub = grpc_mux_->subscribe("foo", {"z"}, callbacks_); + grpc_mux_->resume("bar"); + grpc_mux_->pause("foo"); + auto foo_zz_sub = grpc_mux_->subscribe("foo", {"zz"}, callbacks_); + expectSendMessage("foo", {"zz", "z", "x", "y"}, ""); + grpc_mux_->resume("foo"); + grpc_mux_->pause("foo"); } // Validate behavior when type URL mismatches occur. TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { setup(); - auto invalid_response = std::make_unique(); - FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); + std::unique_ptr invalid_response( + new envoy::api::v2::DiscoveryResponse()); + InSequence s; + auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("type_url_foo", {"x", "y"}, "", true); + expectSendMessage("foo", {"x", "y"}, "", true); grpc_mux_->start(); { - auto response = std::make_unique(); - response->set_type_url("type_url_bar"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + std::unique_ptr response( + new envoy::api::v2::DiscoveryResponse()); + response->set_type_url("bar"); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } { - invalid_response->set_type_url("type_url_foo"); - invalid_response->mutable_resources()->Add()->set_type_url("type_url_bar"); + invalid_response->set_type_url("foo"); + invalid_response->mutable_resources()->Add()->set_type_url("bar"); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)) .WillOnce(Invoke([](Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) { - EXPECT_TRUE( - IsSubstring("", "", - "type URL type_url_bar embedded in an individual Any does not match the " - "message-wide type URL type_url_foo in DiscoveryResponse", - e->what())); + EXPECT_TRUE(IsSubstring( + "", "", "bar does not match the message-wide type URL foo in DiscoveryResponse", + e->what())); })); expectSendMessage( - "type_url_foo", {"x", "y"}, "", false, "", Grpc::Status::WellKnownGrpcStatus::Internal, - fmt::format("type URL type_url_bar embedded in an individual Any does not match the " - "message-wide type URL type_url_foo in DiscoveryResponse {}", + "foo", {"x", "y"}, "", false, "", Grpc::Status::WellKnownGrpcStatus::Internal, + fmt::format("bar does not match the message-wide type URL foo in DiscoveryResponse {}", invalid_response->DebugString())); - grpc_mux_->onDiscoveryResponse(std::move(invalid_response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(invalid_response)); } - expectSendMessage("type_url_foo", {}, ""); + expectSendMessage("foo", {}, ""); } // Validate behavior when watches has an unknown resource name. TEST_F(GrpcMuxImplTest, WildcardWatch) { setup(); + InSequence s; const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - FakeGrpcSubscription foo_sub = makeWatch(type_url, {}); + auto foo_sub = grpc_mux_->subscribe(type_url, {}, callbacks_); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {}, "", true); grpc_mux_->start(); - auto response = std::make_unique(); - response->set_type_url(type_url); - response->set_version_info("1"); - envoy::api::v2::ClusterLoadAssignment load_assignment; - load_assignment.set_cluster_name("x"); - response->add_resources()->PackFrom(load_assignment); - EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) - .WillOnce( - Invoke([&load_assignment](const Protobuf::RepeatedPtrField& resources, - const std::string&) { - EXPECT_EQ(1, resources.size()); - envoy::api::v2::ClusterLoadAssignment expected_assignment; - resources[0].UnpackTo(&expected_assignment); - EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); - })); - expectSendMessage(type_url, {}, "1"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + { + std::unique_ptr response( + new envoy::api::v2::DiscoveryResponse()); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::api::v2::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce( + Invoke([&load_assignment](const Protobuf::RepeatedPtrField& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + envoy::api::v2::ClusterLoadAssignment expected_assignment; + resources[0].UnpackTo(&expected_assignment); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + })); + expectSendMessage(type_url, {}, "1"); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + } } // Validate behavior when watches specify resources (potentially overlapping). TEST_F(GrpcMuxImplTest, WatchDemux) { setup(); - // We will not require InSequence here: an update that causes multiple onConfigUpdates - // causes them in an indeterminate order, based on the whims of the hash map. + InSequence s; const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - NiceMock> foo_callbacks; - FakeGrpcSubscription foo_sub = makeWatch(type_url, {"x", "y"}, foo_callbacks); - NiceMock> bar_callbacks; - FakeGrpcSubscription bar_sub = makeWatch(type_url, {"y", "z"}, bar_callbacks); + NiceMock foo_callbacks; + auto foo_sub = grpc_mux_->subscribe(type_url, {"x", "y"}, foo_callbacks); + NiceMock bar_callbacks; + auto bar_sub = grpc_mux_->subscribe(type_url, {"y", "z"}, bar_callbacks); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); // Should dedupe the "x" resource. expectSendMessage(type_url, {"y", "z", "x"}, "", true); grpc_mux_->start(); - // Send just x; only foo_callbacks should receive an onConfigUpdate(). { - auto response = std::make_unique(); + std::unique_ptr response( + new envoy::api::v2::DiscoveryResponse()); response->set_type_url(type_url); response->set_version_info("1"); envoy::api::v2::ClusterLoadAssignment load_assignment; @@ -324,13 +301,12 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); })); expectSendMessage(type_url, {"y", "z", "x"}, "1"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } - // Send x y and z; foo_ and bar_callbacks should both receive onConfigUpdate()s, carrying {x,y} - // and {y,z} respectively. { - auto response = std::make_unique(); + std::unique_ptr response( + new envoy::api::v2::DiscoveryResponse()); response->set_type_url(type_url); response->set_version_info("2"); envoy::api::v2::ClusterLoadAssignment load_assignment_x; @@ -365,7 +341,7 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment_y)); })); expectSendMessage(type_url, {"y", "z", "x"}, "2"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } expectSendMessage(type_url, {"x", "y"}, "2"); @@ -377,20 +353,21 @@ TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { setup(); InSequence s; const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - NiceMock> foo_callbacks; - FakeGrpcSubscription foo_sub = makeWatch(type_url, {"x", "y"}, foo_callbacks); + NiceMock foo_callbacks; + auto foo_sub = grpc_mux_->subscribe(type_url, {"x", "y"}, foo_callbacks); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {"x", "y"}, "", true); grpc_mux_->start(); - auto response = std::make_unique(); + std::unique_ptr response( + new envoy::api::v2::DiscoveryResponse()); response->set_type_url(type_url); response->set_version_info("1"); EXPECT_CALL(foo_callbacks, onConfigUpdate(_, "1")).Times(0); expectSendMessage(type_url, {"x", "y"}, "1"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); expectSendMessage(type_url, {}, "1"); } @@ -399,14 +376,15 @@ TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { setup(); const std::string& type_url = Config::TypeUrl::get().Cluster; - NiceMock> foo_callbacks; - FakeGrpcSubscription foo_sub = makeWatch(type_url, {}, foo_callbacks); + NiceMock foo_callbacks; + auto foo_sub = grpc_mux_->subscribe(type_url, {}, foo_callbacks); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {}, "", true); grpc_mux_->start(); - auto response = std::make_unique(); + std::unique_ptr response( + new envoy::api::v2::DiscoveryResponse()); response->set_type_url(type_url); response->set_version_info("1"); // Validate that onConfigUpdate is called with empty resources. @@ -414,7 +392,7 @@ TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { .WillOnce(Invoke([](const Protobuf::RepeatedPtrField& resources, const std::string&) { EXPECT_TRUE(resources.empty()); })); expectSendMessage(type_url, {}, "1"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } // Exactly one test requires a mock time system to provoke behavior that cannot @@ -448,15 +426,15 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { for (uint64_t i = 0; i < burst; i++) { std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); - response->set_version_info("type_url_baz"); - response->set_nonce("type_url_bar"); - response->set_type_url("type_url_foo"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + response->set_version_info("baz"); + response->set_nonce("bar"); + response->set_type_url("foo"); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } }; - FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x"}); - expectSendMessage("type_url_foo", {"x"}, "", true); + auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); + expectSendMessage("foo", {"x"}, "", true); grpc_mux_->start(); // Exhausts the limit. @@ -501,24 +479,23 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithEmptyRateLimitSetti for (uint64_t i = 0; i < burst; i++) { std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); - response->set_version_info("type_url_baz"); - response->set_nonce("type_url_bar"); - response->set_type_url("type_url_foo"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + response->set_version_info("baz"); + response->set_nonce("bar"); + response->set_type_url("foo"); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } }; - FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x"}); - expectSendMessage("type_url_foo", {"x"}, "", true); + auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); + expectSendMessage("foo", {"x"}, "", true); grpc_mux_->start(); // Validate that drain_request_timer is enabled when there are no tokens. - EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(100), _)) - .Times(AtLeast(1)); - onReceiveMessage(110); - EXPECT_LE(10, stats_.counter("control_plane.rate_limit_enforced").value()); - EXPECT_LE( - 10, + EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(100), _)); + onReceiveMessage(99); + EXPECT_EQ(1, stats_.counter("control_plane.rate_limit_enforced").value()); + EXPECT_EQ( + 1, stats_.gauge("control_plane.pending_requests", Stats::Gauge::ImportMode::Accumulate).value()); } @@ -558,15 +535,15 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { for (uint64_t i = 0; i < burst; i++) { std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); - response->set_version_info("type_url_baz"); - response->set_nonce("type_url_bar"); - response->set_type_url("type_url_foo"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + response->set_version_info("baz"); + response->set_nonce("bar"); + response->set_type_url("foo"); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } }; - FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x"}); - expectSendMessage("type_url_foo", {"x"}, "", true); + auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); + expectSendMessage("foo", {"x"}, "", true); grpc_mux_->start(); // Validate that rate limit is not enforced for 100 requests. @@ -577,10 +554,10 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(500), _)) .Times(AtLeast(1)); onReceiveMessage(160); - EXPECT_LE(10, stats_.counter("control_plane.rate_limit_enforced").value()); + EXPECT_EQ(12, stats_.counter("control_plane.rate_limit_enforced").value()); Stats::Gauge& pending_requests = stats_.gauge("control_plane.pending_requests", Stats::Gauge::ImportMode::Accumulate); - EXPECT_LE(10, pending_requests.value()); + EXPECT_EQ(12, pending_requests.value()); // Validate that drain requests call when there are multiple requests in queue. time_system_.setMonotonicTime(std::chrono::seconds(10)); @@ -590,7 +567,7 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { EXPECT_EQ(0, pending_requests.value()); } -// Verifies that a message with no resources is accepted. +// Verifies that a message with no resources is accepted. TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { setup(); @@ -602,66 +579,71 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { { // subscribe and unsubscribe to simulate a cluster added and removed expectSendMessage(type_url, {"y"}, "", true); - FakeGrpcSubscription temp_sub = makeWatch(type_url, {"y"}); + auto temp_sub = grpc_mux_->subscribe(type_url, {"y"}, callbacks_); expectSendMessage(type_url, {}, ""); } // simulate the server sending empty CLA message to notify envoy that the CLA was removed. - auto response = std::make_unique(); + std::unique_ptr response( + new envoy::api::v2::DiscoveryResponse()); response->set_nonce("bar"); response->set_version_info("1"); response->set_type_url(type_url); - // Although the update will change nothing for us, we will "accept" it, and so according - // to the spec we should ACK it. - expectSendMessage(type_url, {}, "1", false, "bar"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + // TODO(fredlas) the expectation of no discovery request here is against the xDS spec. + // The upcoming xDS overhaul (part of/followup to PR7293) will fix this. + // + // This contains zero resources. No discovery request should be sent. + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); - // When we become interested in "x", we should send a request indicating that interest. + // when we add the new subscription version should be 1 and nonce should be bar expectSendMessage(type_url, {"x"}, "1", false, "bar"); - FakeGrpcSubscription sub = makeWatch(type_url, {"x"}); - // Watch destroyed -> interest gone -> unsubscribe request. + // simulate a new cluster x is added. add CLA subscription for it. + auto sub = grpc_mux_->subscribe(type_url, {"x"}, callbacks_); expectSendMessage(type_url, {}, "1", false, "bar"); } -// Verifies that a message with some resources is accepted even when there are no watches. -// Rationale: SotW gRPC xDS has always been willing to accept updates that include -// uninteresting resources. It should not matter whether those uninteresting resources -// are accompanied by interesting ones. -// Note: this was previously "rejects", not "accepts". See -// https://github.com/envoyproxy/envoy/pull/8350#discussion_r328218220 for discussion. -TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsResources) { +// Verifies that a message with some resources is rejected when there are no watches. +TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { setup(); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - grpc_mux_->start(); - // subscribe and unsubscribe so that the type is known to envoy - { - expectSendMessage(type_url, {"y"}, "", true); - expectSendMessage(type_url, {}, ""); - FakeGrpcSubscription delete_immediately = makeWatch(type_url, {"y"}); - } - auto response = std::make_unique(); + grpc_mux_->start(); + // subscribe and unsubscribe (by not keeping the return watch) so that the type is known to envoy + expectSendMessage(type_url, {"y"}, "", true); + expectSendMessage(type_url, {}, ""); + grpc_mux_->subscribe(type_url, {"y"}, callbacks_); + + // simulate the server sending CLA message to notify envoy that the CLA was added, + // even though envoy doesn't expect it. Envoy should reject this update. + std::unique_ptr response( + new envoy::api::v2::DiscoveryResponse()); + response->set_nonce("bar"); + response->set_version_info("1"); response->set_type_url(type_url); + envoy::api::v2::ClusterLoadAssignment load_assignment; load_assignment.set_cluster_name("x"); response->add_resources()->PackFrom(load_assignment); - response->set_version_info("1"); - expectSendMessage(type_url, {}, "1"); - grpc_mux_->onDiscoveryResponse(std::move(response)); + // The message should be rejected. + expectSendMessage(type_url, {}, "", false, "bar"); + EXPECT_LOG_CONTAINS("warning", "Ignoring unwatched type URL " + type_url, + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response))); } TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { EXPECT_CALL(local_info_, clusterName()).WillOnce(ReturnRef(EMPTY_STRING)); EXPECT_THROW_WITH_MESSAGE( - GrpcMuxSotw( - std::unique_ptr(async_client_), dispatcher_, + GrpcMuxImpl( + local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, rate_limit_settings_, local_info_, true), + random_, stats_, rate_limit_settings_, true), EnvoyException, "ads: node 'id' and 'cluster' are required. Set it either in 'node' config or via " "--service-node and --service-cluster options."); @@ -670,52 +652,15 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { EXPECT_CALL(local_info_, nodeName()).WillOnce(ReturnRef(EMPTY_STRING)); EXPECT_THROW_WITH_MESSAGE( - GrpcMuxSotw( - std::unique_ptr(async_client_), dispatcher_, + GrpcMuxImpl( + local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, rate_limit_settings_, local_info_, true), + random_, stats_, rate_limit_settings_, true), EnvoyException, "ads: node 'id' and 'cluster' are required. Set it either in 'node' config or via " "--service-node and --service-cluster options."); } - -// Test that we simply ignore a message for an unknown type_url, with no ill effects. -TEST_F(GrpcMuxImplTest, DiscoveryResponseNonexistentSub) { - setup(); - - const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - grpc_mux_->addOrUpdateWatch(type_url, nullptr, {}, callbacks_, std::chrono::milliseconds(0)); - - EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage(type_url, {}, "", true); - grpc_mux_->start(); - { - auto unexpected_response = std::make_unique(); - unexpected_response->set_type_url("unexpected_type_url"); - unexpected_response->set_version_info("0"); - EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "0")).Times(0); - grpc_mux_->onDiscoveryResponse(std::move(unexpected_response)); - } - auto response = std::make_unique(); - response->set_type_url(type_url); - response->set_version_info("1"); - envoy::api::v2::ClusterLoadAssignment load_assignment; - load_assignment.set_cluster_name("x"); - response->add_resources()->PackFrom(load_assignment); - EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) - .WillOnce( - Invoke([&load_assignment](const Protobuf::RepeatedPtrField& resources, - const std::string&) { - EXPECT_EQ(1, resources.size()); - envoy::api::v2::ClusterLoadAssignment expected_assignment; - resources[0].UnpackTo(&expected_assignment); - EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); - })); - expectSendMessage(type_url, {}, "1"); - grpc_mux_->onDiscoveryResponse(std::move(response)); -} - } // namespace } // namespace Config } // namespace Envoy diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index b16a7c3d1799..f7bed5838555 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -22,7 +22,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { EXPECT_CALL(random_, random()); EXPECT_CALL(*timer_, enableTimer(_, _)); subscription_->start({"cluster0", "cluster1"}); - EXPECT_TRUE(statsAre(2, 0, 0, 0, 0, 0)); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); // Ensure this doesn't cause an issue by sending a request, since we don't // have a gRPC stream. subscription_->updateResourceInterest({"cluster2"}); @@ -32,7 +32,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { expectSendMessage({"cluster2"}, "", true); timer_cb_(); - EXPECT_TRUE(statsAre(3, 0, 0, 0, 0, 0)); + EXPECT_TRUE(statsAre(3, 0, 0, 1, 0, 0)); verifyControlPlaneStats(1); } @@ -46,10 +46,8 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { .Times(0); EXPECT_CALL(*timer_, enableTimer(_, _)); EXPECT_CALL(random_, random()); - auto shared_mux = subscription_->getGrpcMuxForTest(); - static_cast(shared_mux.get()) - ->grpcStreamForTest() - .onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Canceled, ""); + subscription_->grpcMux()->grpcStreamForTest().onRemoteClose( + Grpc::Status::WellKnownGrpcStatus::Canceled, ""); EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); verifyControlPlaneStats(0); @@ -60,7 +58,7 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); } -// Validate that when the management server gets multiple requests for the same version, it can +// Validate that When the management server gets multiple requests for the same version, it can // ignore later ones. This allows the nonce to be used. TEST_F(GrpcSubscriptionImplTest, RepeatedNonce) { InSequence s; diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 1c425b50c7e1..476230d68cb5 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -36,22 +36,16 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { : method_descriptor_(Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints")), async_client_(new NiceMock()), timer_(new Event::MockTimer()) { - node_.set_id("node_name"); - node_.set_cluster("cluster_name"); - node_.mutable_locality()->set_zone("zone_name"); - EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); + node_.set_id("fo0"); + EXPECT_CALL(local_info_, node()).WillOnce(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)).WillOnce(Invoke([this](Event::TimerCb timer_cb) { timer_cb_ = timer_cb; return timer_; })); - subscription_ = std::make_unique( - std::make_shared(std::unique_ptr(async_client_), - dispatcher_, *method_descriptor_, random_, stats_store_, - rate_limit_settings_, local_info_, - /*skip_subsequent_node=*/true), - Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, init_fetch_timeout, - /*is_aggregated=*/false); + local_info_, std::unique_ptr(async_client_), dispatcher_, random_, + *method_descriptor_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, + stats_store_, rate_limit_settings_, init_fetch_timeout, true); } ~GrpcSubscriptionTestHarness() override { EXPECT_CALL(async_stream_, sendMessageRaw_(_, false)); } @@ -83,9 +77,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { error_detail->set_code(error_code); error_detail->set_message(error_message); } - EXPECT_CALL( - async_stream_, - sendMessageRaw_(Grpc::ProtoBufferEqIgnoreRepeatedFieldOrdering(expected_request), false)); + EXPECT_CALL(async_stream_, sendMessageRaw_(Grpc::ProtoBufferEq(expected_request), false)); } void startSubscription(const std::set& cluster_names) override { @@ -97,7 +89,8 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { void deliverConfigUpdate(const std::vector& cluster_names, const std::string& version, bool accept) override { - auto response = std::make_unique(); + std::unique_ptr response( + new envoy::api::v2::DiscoveryResponse()); response->set_version_info(version); last_response_nonce_ = std::to_string(HashUtil::xxHash64(version)); response->set_nonce(last_response_nonce_); @@ -122,19 +115,35 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { expectSendMessage(last_cluster_names_, version_, false, Grpc::Status::WellKnownGrpcStatus::Internal, "bad config"); } - auto shared_mux = subscription_->getGrpcMuxForTest(); - static_cast(shared_mux.get())->onDiscoveryResponse(std::move(response)); + subscription_->grpcMux()->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } void updateResourceInterest(const std::set& cluster_names) override { + // The "watch" mechanism means that updates that lose interest in a resource + // will first generate a request for [still watched resources, i.e. without newly unwatched + // ones] before generating the request for all of cluster_names. + // TODO(fredlas) this unnecessary second request will stop happening once the watch mechanism is + // no longer internally used by GrpcSubscriptionImpl. + std::set both; + for (const auto& n : cluster_names) { + if (last_cluster_names_.find(n) != last_cluster_names_.end()) { + both.insert(n); + } + } + expectSendMessage(both, version_); expectSendMessage(cluster_names, version_); subscription_->updateResourceInterest(cluster_names); last_cluster_names_ = cluster_names; } void expectConfigUpdateFailed() override { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)) + .WillOnce([this](ConfigUpdateFailureReason reason, const EnvoyException*) { + if (reason == ConfigUpdateFailureReason::FetchTimedout) { + stats_.init_fetch_timeout_.inc(); + } + }); } void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { diff --git a/test/common/config/new_grpc_mux_impl_test.cc b/test/common/config/new_grpc_mux_impl_test.cc new file mode 100644 index 000000000000..57b1373dab22 --- /dev/null +++ b/test/common/config/new_grpc_mux_impl_test.cc @@ -0,0 +1,117 @@ +#include + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/api/v2/eds.pb.h" + +#include "common/common/empty_string.h" +#include "common/config/new_grpc_mux_impl.h" +#include "common/config/protobuf_link_hacks.h" +#include "common/config/resources.h" +#include "common/config/utility.h" +#include "common/protobuf/protobuf.h" +#include "common/stats/isolated_store_impl.h" + +#include "test/mocks/common.h" +#include "test/mocks/config/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/grpc/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/test_common/logging.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_time.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Invoke; +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Config { +namespace { + +// We test some mux specific stuff below, other unit test coverage for singleton use of +// NewGrpcMuxImpl is provided in [grpc_]subscription_impl_test.cc. +class NewGrpcMuxImplTestBase : public testing::Test { +public: + NewGrpcMuxImplTestBase() + : async_client_(new Grpc::MockAsyncClient()), + control_plane_connected_state_( + stats_.gauge("control_plane.connected_state", Stats::Gauge::ImportMode::NeverImport)) {} + + void setup() { + grpc_mux_ = std::make_unique( + std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + random_, stats_, rate_limit_settings_, local_info_); + } + + NiceMock dispatcher_; + NiceMock random_; + Grpc::MockAsyncClient* async_client_; + NiceMock async_stream_; + std::unique_ptr grpc_mux_; + NiceMock> callbacks_; + NiceMock local_info_; + Stats::IsolatedStoreImpl stats_; + Envoy::Config::RateLimitSettings rate_limit_settings_; + Stats::Gauge& control_plane_connected_state_; +}; + +class NewGrpcMuxImplTest : public NewGrpcMuxImplTestBase { +public: + Event::SimulatedTimeSystem time_system_; +}; + +// TODO(fredlas) #8478 will delete this. +TEST_F(NewGrpcMuxImplTest, JustForCoverageTodoDelete) { + setup(); + EXPECT_TRUE(grpc_mux_->isDelta()); +} + +// Test that we simply ignore a message for an unknown type_url, with no ill effects. +TEST_F(NewGrpcMuxImplTest, DiscoveryResponseNonexistentSub) { + setup(); + + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + grpc_mux_->addOrUpdateWatch(type_url, nullptr, {}, callbacks_, std::chrono::milliseconds(0)); + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + grpc_mux_->start(); + + { + auto unexpected_response = std::make_unique(); + unexpected_response->set_type_url(type_url); + unexpected_response->set_system_version_info("0"); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "0")).Times(0); + grpc_mux_->onDiscoveryResponse(std::move(unexpected_response)); + } + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_system_version_info("1"); + envoy::api::v2::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->mutable_resource()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "1")) + .WillOnce( + Invoke([&load_assignment]( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField&, const std::string&) { + EXPECT_EQ(1, added_resources.size()); + envoy::api::v2::ClusterLoadAssignment expected_assignment; + added_resources[0].resource().UnpackTo(&expected_assignment); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + })); + grpc_mux_->onDiscoveryResponse(std::move(response)); + } +} + +} // namespace +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/sotw_subscription_state_test.cc b/test/common/config/sotw_subscription_state_test.cc deleted file mode 100644 index 2e962598299e..000000000000 --- a/test/common/config/sotw_subscription_state_test.cc +++ /dev/null @@ -1,183 +0,0 @@ -#include "common/config/sotw_subscription_state.h" -#include "common/config/utility.h" -#include "common/stats/isolated_store_impl.h" - -#include "test/mocks/config/mocks.h" -#include "test/mocks/event/mocks.h" -#include "test/mocks/local_info/mocks.h" - -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -using testing::NiceMock; -using testing::Throw; -using testing::UnorderedElementsAre; - -namespace Envoy { -namespace Config { -namespace { - -const char TypeUrl[] = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"; - -class SotwSubscriptionStateTest : public testing::Test { -protected: - SotwSubscriptionStateTest() - : state_(TypeUrl, callbacks_, std::chrono::milliseconds(0U), dispatcher_) { - state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {}); - auto cur_request = getNextDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2", "name3")); - } - - std::unique_ptr getNextDiscoveryRequestAckless() { - auto* ptr = static_cast(state_.getNextRequestAckless()); - return std::unique_ptr(ptr); - } - - UpdateAck deliverDiscoveryResponse(const std::vector& resource_names, - const std::string& version_info, const std::string& nonce) { - envoy::api::v2::DiscoveryResponse response; - response.set_version_info(version_info); - response.set_nonce(nonce); - Protobuf::RepeatedPtrField typed_resources; - for (const auto& resource_name : resource_names) { - envoy::api::v2::ClusterLoadAssignment* load_assignment = typed_resources.Add(); - load_assignment->set_cluster_name(resource_name); - response.add_resources()->PackFrom(*load_assignment); - } - EXPECT_CALL(callbacks_, onConfigUpdate(_, version_info)); - return state_.handleResponse(&response); - } - - UpdateAck deliverBadDiscoveryResponse(const std::string& version_info, const std::string& nonce) { - envoy::api::v2::DiscoveryResponse message; - message.set_version_info(version_info); - message.set_nonce(nonce); - EXPECT_CALL(callbacks_, onConfigUpdate(_, _)).WillOnce(Throw(EnvoyException("oh no"))); - return state_.handleResponse(&message); - } - - NiceMock> callbacks_; - NiceMock dispatcher_; - // We start out interested in three resources: name1, name2, and name3. - SotwSubscriptionState state_; -}; - -// Basic gaining/losing interest in resources should lead to (un)subscriptions. -TEST_F(SotwSubscriptionStateTest, SubscribeAndUnsubscribe) { - { - state_.updateSubscriptionInterest({"name4"}, {"name1"}); - auto cur_request = getNextDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name2", "name3", "name4")); - } - { - state_.updateSubscriptionInterest({"name1"}, {"name3", "name4"}); - auto cur_request = getNextDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2")); - } -} - -// Unlike delta, if SotW gets multiple interest updates before being able to send a request, they -// all collapse to a single update. However, even if the updates all cancel each other out, there -// still will be a request generated. All of the following tests explore different such cases. -TEST_F(SotwSubscriptionStateTest, RemoveThenAdd) { - state_.updateSubscriptionInterest({}, {"name3"}); - state_.updateSubscriptionInterest({"name3"}, {}); - auto cur_request = getNextDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2", "name3")); -} - -TEST_F(SotwSubscriptionStateTest, AddThenRemove) { - state_.updateSubscriptionInterest({"name4"}, {}); - state_.updateSubscriptionInterest({}, {"name4"}); - auto cur_request = getNextDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2", "name3")); -} - -TEST_F(SotwSubscriptionStateTest, AddRemoveAdd) { - state_.updateSubscriptionInterest({"name4"}, {}); - state_.updateSubscriptionInterest({}, {"name4"}); - state_.updateSubscriptionInterest({"name4"}, {}); - auto cur_request = getNextDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names(), - UnorderedElementsAre("name1", "name2", "name3", "name4")); -} - -TEST_F(SotwSubscriptionStateTest, RemoveAddRemove) { - state_.updateSubscriptionInterest({}, {"name3"}); - state_.updateSubscriptionInterest({"name3"}, {}); - state_.updateSubscriptionInterest({}, {"name3"}); - auto cur_request = getNextDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2")); -} - -TEST_F(SotwSubscriptionStateTest, BothAddAndRemove) { - state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); - state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {"name4"}); - state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); - auto cur_request = getNextDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name4")); -} - -TEST_F(SotwSubscriptionStateTest, CumulativeUpdates) { - state_.updateSubscriptionInterest({"name4"}, {}); - state_.updateSubscriptionInterest({"name5"}, {}); - auto cur_request = getNextDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names(), - UnorderedElementsAre("name1", "name2", "name3", "name4", "name5")); -} - -// Verifies that a sequence of good and bad responses from the server all get the appropriate -// ACKs/NACKs from Envoy. -TEST_F(SotwSubscriptionStateTest, AckGenerated) { - // The xDS server's first response includes items for name1 and 2, but not 3. - { - UpdateAck ack = deliverDiscoveryResponse({"name1", "name2"}, "version1", "nonce1"); - EXPECT_EQ("nonce1", ack.nonce_); - EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); - } - // The next response updates 1 and 2, and adds 3. - { - UpdateAck ack = deliverDiscoveryResponse({"name1", "name2", "name3"}, "version2", "nonce2"); - EXPECT_EQ("nonce2", ack.nonce_); - EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); - } - // The next response tries but fails to update all 3, and so should produce a NACK. - { - UpdateAck ack = deliverBadDiscoveryResponse("version3", "nonce3"); - EXPECT_EQ("nonce3", ack.nonce_); - EXPECT_NE(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); - } - // The last response successfully updates all 3. - { - UpdateAck ack = deliverDiscoveryResponse({"name1", "name2", "name3"}, "version4", "nonce4"); - EXPECT_EQ("nonce4", ack.nonce_); - EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); - } -} - -TEST_F(SotwSubscriptionStateTest, CheckUpdatePending) { - // Note that the test fixture ctor causes the first request to be "sent", so we start in the - // middle of a stream, with our initially interested resources having been requested already. - EXPECT_FALSE(state_.subscriptionUpdatePending()); - state_.updateSubscriptionInterest({}, {}); // no change - EXPECT_FALSE(state_.subscriptionUpdatePending()); - state_.markStreamFresh(); - EXPECT_TRUE(state_.subscriptionUpdatePending()); // no change, BUT fresh stream - state_.updateSubscriptionInterest({}, {"name3"}); // one removed - EXPECT_TRUE(state_.subscriptionUpdatePending()); - state_.updateSubscriptionInterest({"name3"}, {}); // one added - EXPECT_TRUE(state_.subscriptionUpdatePending()); -} - -TEST_F(SotwSubscriptionStateTest, handleEstablishmentFailure) { - // Although establishment failure is not supposed to cause an onConfigUpdateFailed() on the - // ultimate actual subscription callbacks, the callbacks reference held is actually to - // the WatchMap, which then calls GrpcSubscriptionImpl(s). It is the GrpcSubscriptionImpl - // that will decline to pass on an onConfigUpdateFailed(ConnectionFailure). - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)); - state_.handleEstablishmentFailure(); -} - -} // namespace -} // namespace Config -} // namespace Envoy diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index 66c37b7c1769..45646de9b531 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -150,17 +150,22 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { if (GetParam() == SubscriptionType::Filesystem) { return; // initial_fetch_timeout not implemented for filesystem. } + InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); + if (GetParam() == SubscriptionType::Http) { + expectDisableInitFetchTimeoutTimer(); + } expectConfigUpdateFailed(); - expectDisableInitFetchTimeoutTimer(); + callInitFetchTimeoutCb(); EXPECT_TRUE(statsAre(1, 0, 0, 0, 1, 0)); } // Validate that initial fetch timer is disabled on config update TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnSuccess) { + InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); @@ -170,6 +175,7 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnSuccess) { // Validate that initial fetch timer is disabled on config update failed TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnFail) { + InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index b7f0214339be..fe6b5e39ab60 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -177,6 +177,54 @@ TEST_F(CdsApiImplTest, ConfigUpdateWith2ValidClusters) { cds_callbacks_->onConfigUpdate(clusters, ""); } +TEST_F(CdsApiImplTest, DeltaConfigUpdate) { + { + InSequence s; + setup(); + } + EXPECT_CALL(initialized_, ready()); + + { + Protobuf::RepeatedPtrField resources; + { + envoy::api::v2::Cluster cluster; + cluster.set_name("cluster_1"); + expectAdd("cluster_1", "v1"); + auto* resource = resources.Add(); + resource->mutable_resource()->PackFrom(cluster); + resource->set_name("cluster_1"); + resource->set_version("v1"); + } + { + envoy::api::v2::Cluster cluster; + cluster.set_name("cluster_2"); + expectAdd("cluster_2", "v1"); + auto* resource = resources.Add(); + resource->mutable_resource()->PackFrom(cluster); + resource->set_name("cluster_2"); + resource->set_version("v1"); + } + cds_callbacks_->onConfigUpdate(resources, {}, "v1"); + } + + { + Protobuf::RepeatedPtrField resources; + { + envoy::api::v2::Cluster cluster; + cluster.set_name("cluster_3"); + expectAdd("cluster_3", "v2"); + auto* resource = resources.Add(); + resource->mutable_resource()->PackFrom(cluster); + resource->set_name("cluster_3"); + resource->set_version("v2"); + } + Protobuf::RepeatedPtrField removed; + *removed.Add() = "cluster_1"; + EXPECT_CALL(cm_, removeCluster(StrEq("cluster_1"))).WillOnce(Return(true)); + cds_callbacks_->onConfigUpdate(resources, removed, "v2"); + } +} + TEST_F(CdsApiImplTest, ConfigUpdateAddsSecondClusterEvenIfFirstThrows) { { InSequence s; diff --git a/test/config/utility.cc b/test/config/utility.cc index 7538ffd9e085..17804e22de83 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -162,6 +162,8 @@ name: envoy.squash nanos: 0 )EOF"; +// TODO(fredlas) set_node_on_first_message_only was true; the delta+SotW unification +// work restores it here. // TODO(#6327) cleaner approach to testing with static config. std::string ConfigHelper::discoveredClustersBootstrap(const std::string& api_type) { return fmt::format( @@ -179,7 +181,7 @@ std::string ConfigHelper::discoveredClustersBootstrap(const std::string& api_typ grpc_services: envoy_grpc: cluster_name: my_cds_cluster - set_node_on_first_message_only: true + set_node_on_first_message_only: false static_resources: clusters: - name: my_cds_cluster diff --git a/test/integration/ads_integration.h b/test/integration/ads_integration.h index c806cd8107e7..31956bd6bb12 100644 --- a/test/integration/ads_integration.h +++ b/test/integration/ads_integration.h @@ -12,6 +12,8 @@ #include "test/common/grpc/grpc_client_integration.h" #include "test/integration/http_integration.h" +// TODO(fredlas) set_node_on_first_message_only was true; the delta+SotW unification +// work restores it here. namespace Envoy { static std::string AdsIntegrationConfig(const std::string& api_type) { // Note: do not use CONSTRUCT_ON_FIRST_USE here! @@ -23,7 +25,7 @@ static std::string AdsIntegrationConfig(const std::string& api_type) { ads: {{}} ads_config: api_type: {} - set_node_on_first_message_only: true + set_node_on_first_message_only: false static_resources: clusters: name: dummy_cluster diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 6842ae951470..56e7b33be962 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -55,7 +55,7 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest( - Config::TypeUrl::get().Cluster, "", {}, {}, {}, false, + Config::TypeUrl::get().Cluster, "", {}, {}, {}, true, Grpc::Status::WellKnownGrpcStatus::Internal, fmt::format("does not match the message-wide type URL {}", Config::TypeUrl::get().Cluster))); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, @@ -70,7 +70,7 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", - {"cluster_0"}, {}, {}, false, + {"cluster_0"}, {}, {}, true, Grpc::Status::WellKnownGrpcStatus::Internal, fmt::format("does not match the message-wide type URL {}", Config::TypeUrl::get().ClusterLoadAssignment))); @@ -85,7 +85,7 @@ TEST_P(AdsIntegrationTest, Failure) { {buildRouteConfig("listener_0", "route_config_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest( - Config::TypeUrl::get().Listener, "", {}, {}, {}, false, + Config::TypeUrl::get().Listener, "", {}, {}, {}, true, Grpc::Status::WellKnownGrpcStatus::Internal, fmt::format("does not match the message-wide type URL {}", Config::TypeUrl::get().Listener))); sendDiscoveryResponse( @@ -100,7 +100,7 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", - {"route_config_0"}, {}, {}, false, + {"route_config_0"}, {}, {}, true, Grpc::Status::WellKnownGrpcStatus::Internal, fmt::format("does not match the message-wide type URL {}", Config::TypeUrl::get().RouteConfiguration))); @@ -364,11 +364,12 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); // CDS is resumed and EDS response was acknowledged. - // - // Envoy will ACK both Cluster messages. Since they arrived while CDS was paused, they aren't - // sent until CDS is unpaused. Since version 3 has already arrived by the time the version 2 - // ACK goes out, they're both acknowledging version 3. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); + if (sotw_or_delta_ == Grpc::SotwOrDelta::Delta) { + // Envoy will ACK both Cluster messages. Since they arrived while CDS was paused, they aren't + // sent until CDS is unpaused. Since version 3 has already arrived by the time the version 2 + // ACK goes out, they're both acknowledging version 3. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); + } EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "2", {"warming_cluster_2", "warming_cluster_1"}, {}, {})); diff --git a/test/integration/integration.cc b/test/integration/integration.cc index 40d4a48cef53..79c2cd7ae0ea 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -568,27 +568,11 @@ AssertionResult BaseIntegrationTest::compareDiscoveryRequest( expect_node, expected_error_code, expected_error_substring); } else { return compareDeltaDiscoveryRequest(expected_type_url, expected_resource_names_added, - expected_resource_names_removed, expect_node, - expected_error_code, expected_error_substring); + expected_resource_names_removed, expected_error_code, + expected_error_substring); } } -AssertionResult compareSets(const std::set& set1, const std::set& set2, - absl::string_view name) { - if (set1 == set2) { - return AssertionSuccess(); - } - auto failure = AssertionFailure() << name << " field not as expected.\nExpected: {"; - for (const auto& x : set1) { - failure << x << ", "; - } - failure << "}\nActual: {"; - for (const auto& x : set2) { - failure << x << ", "; - } - return failure << "}"; -} - AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, const std::vector& expected_resource_names, bool expect_node, @@ -600,58 +584,64 @@ AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( EXPECT_TRUE(discovery_request.has_node()); EXPECT_FALSE(discovery_request.node().id().empty()); EXPECT_FALSE(discovery_request.node().cluster().empty()); + } else { + EXPECT_FALSE(discovery_request.has_node()); } if (expected_type_url != discovery_request.type_url()) { return AssertionFailure() << fmt::format("type_url {} does not match expected {}", discovery_request.type_url(), expected_type_url); } - - // Sort to ignore ordering. - std::set expected_sub{expected_resource_names.begin(), - expected_resource_names.end()}; - std::set actual_sub{discovery_request.resource_names().cbegin(), - discovery_request.resource_names().cend()}; - auto sub_result = compareSets(expected_sub, actual_sub, "resource_names"); - if (!sub_result) { - return sub_result; + if (!(expected_error_code == discovery_request.error_detail().code())) { + return AssertionFailure() << fmt::format("error_code {} does not match expected {}", + discovery_request.error_detail().code(), + expected_error_code); + } + EXPECT_TRUE( + IsSubstring("", "", expected_error_substring, discovery_request.error_detail().message())); + const std::vector resource_names(discovery_request.resource_names().cbegin(), + discovery_request.resource_names().cend()); + if (expected_resource_names != resource_names) { + return AssertionFailure() << fmt::format( + "resources {} do not match expected {} in {}", absl::StrJoin(resource_names, ","), + absl::StrJoin(expected_resource_names, ","), discovery_request.DebugString()); } if (expected_version != discovery_request.version_info()) { return AssertionFailure() << fmt::format("version {} does not match expected {} in {}", discovery_request.version_info(), expected_version, discovery_request.DebugString()); } - if (discovery_request.error_detail().code() != expected_error_code) { - return AssertionFailure() << fmt::format( - "error code {} does not match expected {}. (Error message is {}).", - discovery_request.error_detail().code(), expected_error_code, - discovery_request.error_detail().message()); + return AssertionSuccess(); +} + +AssertionResult compareSets(const std::set& set1, const std::set& set2, + absl::string_view name) { + if (set1 == set2) { + return AssertionSuccess(); } - if (expected_error_code != Grpc::Status::WellKnownGrpcStatus::Ok && - discovery_request.error_detail().message().find(expected_error_substring) == - std::string::npos) { - return AssertionFailure() << "\"" << expected_error_substring - << "\" is not a substring of actual error message \"" - << discovery_request.error_detail().message() << "\""; + auto failure = AssertionFailure() << name << " field not as expected.\nExpected: {"; + for (const auto& x : set1) { + failure << x << ", "; } - return AssertionSuccess(); + failure << "}\nActual: {"; + for (const auto& x : set2) { + failure << x << ", "; + } + return failure << "}"; } AssertionResult BaseIntegrationTest::compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, const std::vector& expected_resource_unsubscriptions, FakeStreamPtr& xds_stream, - bool expect_node, const Protobuf::int32 expected_error_code, - const std::string& expected_error_substring) { + const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { envoy::api::v2::DeltaDiscoveryRequest request; VERIFY_ASSERTION(xds_stream->waitForGrpcMessage(*dispatcher_, request)); - if (expect_node) { - EXPECT_TRUE(request.has_node()); - EXPECT_FALSE(request.node().id().empty()); - EXPECT_FALSE(request.node().cluster().empty()); + // Verify all we care about node. + if (!request.has_node() || request.node().id().empty() || request.node().cluster().empty()) { + return AssertionFailure() << "Weird node field"; } - if (request.type_url() != expected_type_url) { return AssertionFailure() << fmt::format("type_url {} does not match expected {}.", request.type_url(), expected_type_url); @@ -665,11 +655,12 @@ AssertionResult BaseIntegrationTest::compareDeltaDiscoveryRequest( request.resource_names_subscribe().end()}; std::set actual_unsub{request.resource_names_unsubscribe().begin(), request.resource_names_unsubscribe().end()}; - auto sub_result = compareSets(expected_sub, actual_sub, "resource_names_subscribe"); + auto sub_result = compareSets(expected_sub, actual_sub, "expected_resource_subscriptions"); if (!sub_result) { return sub_result; } - auto unsub_result = compareSets(expected_unsub, actual_unsub, "resource_names_unsubscribe"); + auto unsub_result = + compareSets(expected_unsub, actual_unsub, "expected_resource_unsubscriptions"); if (!unsub_result) { return unsub_result; } diff --git a/test/integration/integration.h b/test/integration/integration.h index 8c07b6499b7c..66461a88a1c7 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -220,13 +220,15 @@ class BaseIntegrationTest : Logger::Loggable { // sending/receiving to/from the (imaginary) xDS server. You should almost always use // compareDiscoveryRequest() and sendDiscoveryResponse(), but the SotW/delta-specific versions are // available if you're writing a SotW/delta-specific test. + // TODO(fredlas) expect_node was defaulting false here; the delta+SotW unification work restores + // it. AssertionResult compareDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, const std::vector& expected_resource_names, const std::vector& expected_resource_names_added, - const std::vector& expected_resource_names_removed, bool expect_node = false, + const std::vector& expected_resource_names_removed, bool expect_node = true, const Protobuf::int32 expected_error_code = Grpc::Status::WellKnownGrpcStatus::Ok, - const std::string& expected_error_substring = ""); + const std::string& expected_error_message = ""); template void sendDiscoveryResponse(const std::string& type_url, const std::vector& state_of_the_world, const std::vector& added_or_updated, @@ -241,26 +243,28 @@ class BaseIntegrationTest : Logger::Loggable { AssertionResult compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, - const std::vector& expected_resource_unsubscriptions, bool expect_node = false, + const std::vector& expected_resource_unsubscriptions, const Protobuf::int32 expected_error_code = Grpc::Status::WellKnownGrpcStatus::Ok, - const std::string& expected_error_substring = "") { + const std::string& expected_error_message = "") { return compareDeltaDiscoveryRequest(expected_type_url, expected_resource_subscriptions, - expected_resource_unsubscriptions, xds_stream_, expect_node, - expected_error_code, expected_error_substring); + expected_resource_unsubscriptions, xds_stream_, + expected_error_code, expected_error_message); } AssertionResult compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, const std::vector& expected_resource_unsubscriptions, FakeStreamPtr& stream, - bool expect_node = false, const Protobuf::int32 expected_error_code = Grpc::Status::WellKnownGrpcStatus::Ok, - const std::string& expected_error_substring = ""); + const std::string& expected_error_message = ""); + + // TODO(fredlas) expect_node was defaulting false here; the delta+SotW unification work restores + // it. AssertionResult compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, - const std::vector& expected_resource_names, bool expect_node = false, + const std::vector& expected_resource_names, bool expect_node = true, const Protobuf::int32 expected_error_code = Grpc::Status::WellKnownGrpcStatus::Ok, - const std::string& expected_error_substring = ""); + const std::string& expected_error_message = ""); template void sendSotwDiscoveryResponse(const std::string& type_url, const std::vector& messages, diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc index 3ffcc5cf54a5..6f223f1a626d 100644 --- a/test/integration/rtds_integration_test.cc +++ b/test/integration/rtds_integration_test.cc @@ -6,6 +6,8 @@ namespace Envoy { namespace { +// TODO(fredlas) set_node_on_first_message_only was true; the delta+SotW unification +// work restores it here. std::string tdsBootstrapConfig(absl::string_view api_type) { return fmt::format(R"EOF( static_resources: @@ -36,7 +38,7 @@ std::string tdsBootstrapConfig(absl::string_view api_type) { grpc_services: envoy_grpc: cluster_name: rtds_cluster - set_node_on_first_message_only: true + set_node_on_first_message_only: false - name: some_admin_layer admin_layer: {{}} admin: diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 484c07239964..3caca65ec896 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -82,7 +82,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, scoped_routes->mutable_scoped_rds() ->mutable_scoped_rds_config_source() ->mutable_api_config_source(); - if (sotwOrDelta() == Grpc::SotwOrDelta::Delta) { + if (isDelta()) { srds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); } else { srds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); @@ -169,7 +169,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, const std::vector& to_add_list, const std::vector& to_delete_list, const std::string& version) { - if (sotwOrDelta() == Grpc::SotwOrDelta::Delta) { + if (isDelta()) { sendDeltaScopedRdsResponse(to_add_list, to_delete_list, version); } else { sendSotwScopedRdsResponse(sotw_list, version); @@ -217,6 +217,8 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, response); } + bool isDelta() { return sotwOrDelta() == Grpc::SotwOrDelta::Delta; } + const std::string srds_config_name_{"foo-scoped-routes"}; FakeUpstreamInfo scoped_rds_upstream_info_; FakeUpstreamInfo rds_upstream_info_; @@ -283,7 +285,7 @@ route_configuration_name: {} } test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", // update_attempt only increase after a response - sotwOrDelta() == Grpc::SotwOrDelta::Delta ? 1 : 2); + isDelta() ? 1 : 2); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 1); // The version gauge should be set to xxHash64("1"). test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index 19b4a3ec0d98..4a3e3099e0c1 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -25,11 +25,27 @@ MockSubscriptionFactory::MockSubscriptionFactory() { MockSubscriptionFactory::~MockSubscriptionFactory() = default; +MockGrpcMuxWatch::MockGrpcMuxWatch() = default; +MockGrpcMuxWatch::~MockGrpcMuxWatch() { cancel(); } + MockGrpcMux::MockGrpcMux() = default; MockGrpcMux::~MockGrpcMux() = default; MockGrpcStreamCallbacks::MockGrpcStreamCallbacks() = default; MockGrpcStreamCallbacks::~MockGrpcStreamCallbacks() = default; +GrpcMuxWatchPtr MockGrpcMux::subscribe(const std::string& type_url, + const std::set& resources, + GrpcMuxCallbacks& callbacks) { + return GrpcMuxWatchPtr(subscribe_(type_url, resources, callbacks)); +} + +MockGrpcMuxCallbacks::MockGrpcMuxCallbacks() { + ON_CALL(*this, resourceName(testing::_)) + .WillByDefault(testing::Invoke(TestUtility::xdsResourceName)); +} + +MockGrpcMuxCallbacks::~MockGrpcMuxCallbacks() = default; + } // namespace Config } // namespace Envoy diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index ee4bae564f76..7fdf47552197 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -62,21 +62,53 @@ class MockSubscriptionFactory : public SubscriptionFactory { SubscriptionCallbacks* callbacks_{}; }; +class MockGrpcMuxWatch : public GrpcMuxWatch { +public: + MockGrpcMuxWatch(); + ~MockGrpcMuxWatch() override; + + MOCK_METHOD0(cancel, void()); +}; + class MockGrpcMux : public GrpcMux { public: MockGrpcMux(); ~MockGrpcMux() override; MOCK_METHOD0(start, void()); + MOCK_METHOD3(subscribe_, + GrpcMuxWatch*(const std::string& type_url, const std::set& resources, + GrpcMuxCallbacks& callbacks)); + GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, + GrpcMuxCallbacks& callbacks) override; + MOCK_METHOD1(pause, void(const std::string& type_url)); + MOCK_METHOD1(resume, void(const std::string& type_url)); + MOCK_CONST_METHOD1(paused, bool(const std::string& type_url)); + + MOCK_METHOD5(addSubscription, + void(const std::set& resources, const std::string& type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats& stats, + std::chrono::milliseconds init_fetch_timeout)); + MOCK_METHOD2(updateResourceInterest, + void(const std::set& resources, const std::string& type_url)); + MOCK_METHOD5(addOrUpdateWatch, Watch*(const std::string& type_url, Watch* watch, const std::set& resources, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout)); MOCK_METHOD2(removeWatch, void(const std::string& type_url, Watch* watch)); - MOCK_METHOD1(pause, void(const std::string& type_url)); - MOCK_METHOD1(resume, void(const std::string& type_url)); - MOCK_CONST_METHOD1(paused, bool(const std::string& type_url)); - MOCK_METHOD0(disableInitFetchTimeoutTimer, void()); +}; + +class MockGrpcMuxCallbacks : public GrpcMuxCallbacks { +public: + MockGrpcMuxCallbacks(); + ~MockGrpcMuxCallbacks() override; + + MOCK_METHOD2(onConfigUpdate, void(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info)); + MOCK_METHOD2(onConfigUpdateFailed, + void(Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e)); + MOCK_METHOD1(resourceName, std::string(const ProtobufWkt::Any& resource)); }; class MockGrpcStreamCallbacks : public GrpcStreamCallbacks { diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index ffa81364045e..e5843f0bc64b 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -139,7 +139,6 @@ MockClusterManager::MockClusterManager() { ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault(ReturnRef(async_client_)); ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault((ReturnRef(async_client_))); ON_CALL(*this, bindConfig()).WillByDefault(ReturnRef(bind_config_)); - ads_mux_ = std::make_shared>(); ON_CALL(*this, adsMux()).WillByDefault(Return(ads_mux_)); ON_CALL(*this, grpcAsyncClientManager()).WillByDefault(ReturnRef(async_client_manager_)); ON_CALL(*this, localClusterName()).WillByDefault((ReturnRef(local_cluster_name_))); diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 3447223bfcce..76a9f05aadc1 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -312,7 +312,6 @@ absl accesslog accessor accessors -ackless acks acls addr @@ -553,7 +552,6 @@ genrule getaddrinfo getaffinity getifaddrs -getNextRequestAckless getpeername getsockname getsockopt @@ -771,7 +769,6 @@ pkey plaintext pluggable pointee -polymorphism popen pos posix @@ -915,8 +912,6 @@ sockfd sockopt sockopts somestring -SotW -SotwSubscriptionState spanid spdlog splitter @@ -955,7 +950,6 @@ submessages subnet subnets suboptimal -SubscriptionStateFactory subsecond subseconds subsequence From 4787e0a9b6810ff4cc0dd3f8d78cb8a1d7c2d73a Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Sun, 17 Nov 2019 07:06:18 -0800 Subject: [PATCH 02/29] build: update protobuf to 3.10.1 (#9040) Involves zlib BUILD move to remove the bind. Risk Level: Low Signed-off-by: Lizan Zhou --- bazel/external/apache_thrift.BUILD | 2 +- bazel/protobuf.patch | 97 ++++++++++++++++++---- bazel/repositories.bzl | 29 ++----- bazel/repository_locations.bzl | 26 +++--- test/integration/stats_integration_test.cc | 11 ++- test/mocks/grpc/mocks.h | 2 +- 6 files changed, 108 insertions(+), 59 deletions(-) diff --git a/bazel/external/apache_thrift.BUILD b/bazel/external/apache_thrift.BUILD index 210c36cdf6c3..02cbf535514d 100644 --- a/bazel/external/apache_thrift.BUILD +++ b/bazel/external/apache_thrift.BUILD @@ -23,5 +23,5 @@ py_library( name = "apache_thrift", srcs = [":thrift_files"], visibility = ["//visibility:public"], - deps = ["@six_archive//:six"], + deps = ["@six"], ) diff --git a/bazel/protobuf.patch b/bazel/protobuf.patch index d00e39ef9412..cb7637afe11b 100644 --- a/bazel/protobuf.patch +++ b/bazel/protobuf.patch @@ -1,33 +1,94 @@ +# https://github.com/protocolbuffers/protobuf/pull/6720 +diff --git a/third_party/BUILD b/third_party/BUILD +new file mode 100644 +index 0000000000..b66101a39a +--- /dev/null ++++ b/third_party/BUILD +@@ -0,0 +1 @@ ++exports_files(["six.BUILD", "zlib.BUILD"]) + +# https://github.com/protocolbuffers/protobuf/pull/6896 diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc -index 3844fa6b8b..5486887295 100644 +index 62b3f0a871..bb3df47ccf 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc -@@ -1065,10 +1065,12 @@ char* FastUInt32ToBufferLeft(uint32 u, char* buffer) { +@@ -1435,32 +1435,44 @@ AlphaNum::AlphaNum(strings::Hex hex) { + // after the area just overwritten. It comes in multiple flavors to minimize + // call overhead. + static char *Append1(char *out, const AlphaNum &x) { +- memcpy(out, x.data(), x.size()); +- return out + x.size(); ++ if (x.size() > 0) { ++ memcpy(out, x.data(), x.size()); ++ out += x.size(); ++ } ++ return out; + } + + static char *Append2(char *out, const AlphaNum &x1, const AlphaNum &x2) { +- memcpy(out, x1.data(), x1.size()); +- out += x1.size(); +- +- memcpy(out, x2.data(), x2.size()); +- return out + x2.size(); ++ if (x1.size() > 0) { ++ memcpy(out, x1.data(), x1.size()); ++ out += x1.size(); ++ } ++ if (x2.size() > 0) { ++ memcpy(out, x2.data(), x2.size()); ++ out += x2.size(); ++ } ++ return out; } - char* FastInt32ToBufferLeft(int32 i, char* buffer) { -- uint32 u = i; -+ uint32 u = 0; - if (i < 0) { - *buffer++ = '-'; -- u = -i; -+ u -= i; -+ } else { -+ u = i; - } - return FastUInt32ToBufferLeft(u, buffer); +-static char *Append4(char *out, +- const AlphaNum &x1, const AlphaNum &x2, ++static char *Append4(char *out, const AlphaNum &x1, const AlphaNum &x2, + const AlphaNum &x3, const AlphaNum &x4) { +- memcpy(out, x1.data(), x1.size()); +- out += x1.size(); +- +- memcpy(out, x2.data(), x2.size()); +- out += x2.size(); +- +- memcpy(out, x3.data(), x3.size()); +- out += x3.size(); +- +- memcpy(out, x4.data(), x4.size()); +- return out + x4.size(); ++ if (x1.size() > 0) { ++ memcpy(out, x1.data(), x1.size()); ++ out += x1.size(); ++ } ++ if (x2.size() > 0) { ++ memcpy(out, x2.data(), x2.size()); ++ out += x2.size(); ++ } ++ if (x3.size() > 0) { ++ memcpy(out, x3.data(), x3.size()); ++ out += x3.size(); ++ } ++ if (x4.size() > 0) { ++ memcpy(out, x4.data(), x4.size()); ++ out += x4.size(); ++ } ++ return out; } + + string StrCat(const AlphaNum &a, const AlphaNum &b) { +# patching for zlib binding diff --git a/BUILD b/BUILD -index 6665de94..55f28582 100644 +index efc3d8e7f..746ad4851 100644 --- a/BUILD +++ b/BUILD -@@ -19,7 +19,7 @@ config_setting( +@@ -24,7 +24,7 @@ config_setting( # ZLIB configuration ################################################################################ - + -ZLIB_DEPS = ["@zlib//:zlib"] +ZLIB_DEPS = ["//external:zlib"] - + ################################################################################ - # Protobuf Runtime Library + # Protobuf Runtime Library \ No newline at end of file diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 89d679447cf3..b42eb34074ba 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -66,12 +66,8 @@ def _python_deps(): build_file = "@envoy//bazel/external:twitter_common_finagle_thrift.BUILD", ) _repository_impl( - name = "six_archive", - build_file = "@com_google_protobuf//:six.BUILD", - ) - native.bind( name = "six", - actual = "@six_archive//:six", + build_file = "@com_google_protobuf//third_party:six.BUILD", ) # Bazel native C++ dependencies. For the dependencies that doesn't provide autoconf/automake builds. @@ -321,6 +317,7 @@ def _net_zlib(): build_file_content = BUILD_ALL_CONTENT, **location ) + native.bind( name = "zlib", actual = "@envoy//bazel/foreign_cc:zlib", @@ -514,27 +511,10 @@ def _com_google_absl(): def _com_google_protobuf(): _repository_impl( "com_google_protobuf", - # The patch includes - # https://github.com/protocolbuffers/protobuf/pull/6333 and also uses - # foreign_cc build for zlib as its dependency. - # TODO(asraa): remove this when protobuf 3.10 is released. - patch_args = ["-p1"], patches = ["@envoy//bazel:protobuf.patch"], - ) - - # Needed for cc_proto_library, Bazel doesn't support aliases today for repos, - # see https://groups.google.com/forum/#!topic/bazel-discuss/859ybHQZnuI and - # https://github.com/bazelbuild/bazel/issues/3219. - _repository_impl( - "com_google_protobuf_cc", - repository_key = "com_google_protobuf", - # The patch includes - # https://github.com/protocolbuffers/protobuf/pull/6333 and also uses - # foreign_cc build for zlib as its dependency. - # TODO(asraa): remove this when protobuf 3.10 is released. patch_args = ["-p1"], - patches = ["@envoy//bazel:protobuf.patch"], ) + native.bind( name = "protobuf", actual = "@com_google_protobuf//:protobuf", @@ -549,7 +529,7 @@ def _com_google_protobuf(): ) native.bind( name = "protoc", - actual = "@com_google_protobuf_cc//:protoc", + actual = "@com_google_protobuf//:protoc", ) # Needed for `bazel fetch` to work with @com_google_protobuf @@ -758,6 +738,7 @@ def _foreign_cc_dependencies(): def _rules_proto_dependencies(): _repository_impl("rules_proto") + _repository_impl("rules_python") def _is_linux(ctxt): return ctxt.os.name == "linux" diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index cd84eca89a07..85d8923359ae 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -223,9 +223,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/google/googletest/archive/d7003576dd133856432e2e07340f45926242cc3a.tar.gz"], ), com_google_protobuf = dict( - sha256 = "7c99ddfe0227cbf6a75d1e75b194e0db2f672d2d2ea88fb06bdc83fe0af4c06d", - strip_prefix = "protobuf-3.9.2", - urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v3.9.2/protobuf-all-3.9.2.tar.gz"], + sha256 = "d7cfd31620a352b2ee8c1ed883222a0d77e44346643458e062e86b1d069ace3e", + strip_prefix = "protobuf-3.10.1", + urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v3.10.1/protobuf-all-3.10.1.tar.gz"], ), grpc_httpjson_transcoding = dict( sha256 = "a447458b47ea4dc1d31499f555769af437c5d129d988ec1e13d5fdd0a6a36b4e", @@ -244,17 +244,21 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/rules_foreign_cc/archive/7bc4be735b0560289f6b86ab6136ee25d20b65b7.tar.gz"], ), rules_proto = dict( - sha256 = "602e7161d9195e50246177e7c55b2f39950a9cf7366f74ed5f22fd45750cd208", - strip_prefix = "rules_proto-97d8af4dc474595af3900dd85cb3a29ad28cc313", - # 2019-08-02 + sha256 = "296ffd3e7992bd83fa75151255f7c7f27d22d6e52e2fd3c3d3d10c292317fbed", + strip_prefix = "rules_proto-f6c112fa4eb2b8f934feb938a6fce41425e41587", + # 2019-11-07 urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/97d8af4dc474595af3900dd85cb3a29ad28cc313.tar.gz", - "https://github.com/bazelbuild/rules_proto/archive/97d8af4dc474595af3900dd85cb3a29ad28cc313.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/f6c112fa4eb2b8f934feb938a6fce41425e41587.tar.gz", + "https://github.com/bazelbuild/rules_proto/archive/f6c112fa4eb2b8f934feb938a6fce41425e41587.tar.gz", ], ), - six_archive = dict( - sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", - urls = ["https://files.pythonhosted.org/packages/b3/b2/238e2590826bfdd113244a40d9d3eb26918bd798fc187e2360a8367068db/six-1.10.0.tar.gz"], + rules_python = dict( + sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161", + urls = ["https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz"], + ), + six = dict( + sha256 = "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73", + urls = ["https://files.pythonhosted.org/packages/dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca/six-1.12.0.tar.gz"], ), io_opencensus_cpp = dict( sha256 = "8078195ce90925c142f5c030b9681771db7b7554ebe2156b08848adeb006c40e", diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index e270cd225b6b..73b2303af4a6 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -258,6 +258,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // 2019/10/17 8537 43308 44000 add new enum value HTTP3 // 2019/10/17 8484 43340 44000 stats: add unit support to histogram // 2019/11/01 8859 43563 44000 build: switch to libc++ by default + // 2019/11/15 9040 43371 44000 build: update protobuf to 3.10.1 // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -271,7 +272,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // If you encounter a failure here, please see // https://github.com/envoyproxy/envoy/blob/master/source/docs/stats.md#stats-memory-tests // for details on how to fix. - EXPECT_MEMORY_EQ(m_per_cluster, 43563); // 104 bytes higher than a debug build. + EXPECT_MEMORY_EQ(m_per_cluster, 43371); // 104 bytes higher than a debug build. EXPECT_MEMORY_LE(m_per_cluster, 44000); } @@ -303,6 +304,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // 2019/10/17 8537 34966 35000 add new enum value HTTP3 // 2019/10/17 8484 34998 35000 stats: add unit support to histogram // 2019/11/01 8859 35221 36000 build: switch to libc++ by default + // 2019/11/15 9040 35029 35500 build: update protobuf to 3.10.1 // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -316,8 +318,8 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // If you encounter a failure here, please see // https://github.com/envoyproxy/envoy/blob/master/source/docs/stats.md#stats-memory-tests // for details on how to fix. - EXPECT_MEMORY_EQ(m_per_cluster, 35221); // 104 bytes higher than a debug build. - EXPECT_MEMORY_LE(m_per_cluster, 36000); + EXPECT_MEMORY_EQ(m_per_cluster, 35029); // 104 bytes higher than a debug build. + EXPECT_MEMORY_LE(m_per_cluster, 35500); } TEST_P(ClusterMemoryTestRunner, MemoryLargeHostSizeWithStats) { @@ -344,6 +346,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeHostSizeWithStats) { // 2019/09/10 8216 1283 1315 Use primitive counters for host stats // 2019/11/01 8859 1299 1315 build: switch to libc++ by default // 2019/11/12 8998 1299 1350 test: adjust memory limit for macOS + // 2019/11/15 9040 1283 1350 build: update protobuf to 3.10.1 // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -353,7 +356,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeHostSizeWithStats) { // If you encounter a failure here, please see // https://github.com/envoyproxy/envoy/blob/master/source/docs/stats.md#stats-memory-tests // for details on how to fix. - EXPECT_MEMORY_EQ(m_per_host, 1299); + EXPECT_MEMORY_EQ(m_per_host, 1283); EXPECT_MEMORY_LE(m_per_host, 1350); } diff --git a/test/mocks/grpc/mocks.h b/test/mocks/grpc/mocks.h index 06d24a319e45..e94e7785abb7 100644 --- a/test/mocks/grpc/mocks.h +++ b/test/mocks/grpc/mocks.h @@ -103,7 +103,7 @@ class MockAsyncClientManager : public AsyncClientManager { MATCHER_P(ProtoBufferEq, expected, "") { typename std::remove_const::type proto; - if (!proto.ParseFromArray(static_cast(arg->linearize(arg->length())), arg->length())) { + if (!proto.ParseFromString(arg->toString())) { *result_listener << "\nParse of buffer failed\n"; return false; } From b0cbc879fc643d54d0309450bc2c233b3fa3b8fa Mon Sep 17 00:00:00 2001 From: Alvin Baptiste Date: Mon, 18 Nov 2019 10:49:59 -0800 Subject: [PATCH 03/29] http: sanitize headers nominated by the Connection header (#8862) Description: Sanitize headers nominated by the Connection header Risk Level: Medium Testing: Added several test cases to verify the header manipulation. Also executed bazel test //test/... Docs Changes: N/A Release Notes: N/A Fixes: #8623 Signed-off-by: Alvin Baptiste --- docs/root/intro/version_history.rst | 1 + source/common/http/http1/codec_impl.cc | 13 +- source/common/http/http1/codec_impl.h | 3 +- source/common/http/utility.cc | 128 ++++++++- source/common/http/utility.h | 9 + source/common/runtime/runtime_features.cc | 1 + test/common/http/utility_test.cc | 274 ++++++++++++++++++++ test/integration/header_integration_test.cc | 70 +++++ 8 files changed, 495 insertions(+), 4 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index cd62cf95ce3b..20813ba2ca2a 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -9,6 +9,7 @@ Version history * build: official released binary is now built against libc++. * ext_authz: added :ref:`configurable ability` to send the :ref:`certificate` to the `ext_authz` service. * health check: gRPC health checker sets the gRPC deadline to the configured timeout duration. +* http: added the ability to sanitize headers nominated by the Connection header. This new behavior is guarded by envoy.reloadable_features.connection_header_sanitization which defaults to true. * http: support :ref:`auto_host_rewrite_header` in the dynamic forward proxy. * jwt_authn: added :ref:`bypass_cors_preflight` to allow bypassing the CORS preflight request. * lb_subset_config: new fallback policy for selectors: :ref:`KEYS_SUBSET` diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index 900776e18dfe..c0e1af24a69a 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -360,7 +360,9 @@ ConnectionImpl::ConnectionImpl(Network::Connection& connection, Stats::Scope& st [&]() -> void { this->onAboveHighWatermark(); }), max_headers_kb_(max_headers_kb), max_headers_count_(max_headers_count), strict_header_validation_( - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.strict_header_validation")) { + Runtime::runtimeFeatureEnabled("envoy.reloadable_features.strict_header_validation")), + connection_header_sanitization_(Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.connection_header_sanitization")) { output_buffer_.setWatermarks(connection.bufferLimit()); http_parser_init(&parser_, type); parser_.data = this; @@ -532,6 +534,15 @@ int ConnectionImpl::onHeadersCompleteBase() { ENVOY_CONN_LOG(trace, "codec entering upgrade mode.", connection_); handling_upgrade_ = true; } + } else if (connection_header_sanitization_ && current_header_map_->Connection()) { + // If we fail to sanitize the request, return a 400 to the client + if (!Utility::sanitizeConnectionHeader(*current_header_map_)) { + absl::string_view header_value = current_header_map_->Connection()->value().getStringView(); + ENVOY_CONN_LOG(debug, "Invalid nominated headers in Connection: {}", connection_, + header_value); + error_code_ = Http::Code::BadRequest; + sendProtocolError(); + } } int rc = onHeadersComplete(std::move(current_header_map_)); diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index 8b00bdc6358c..717e3384e027 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -308,7 +308,8 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggablevalue(); + + StringUtil::CaseUnorderedSet headers_to_remove{}; + std::vector connection_header_tokens = + StringUtil::splitToken(connection_header_value.getStringView(), ",", false); + + // If we have 10 or more nominated headers, fail this request + if (connection_header_tokens.size() >= MAX_ALLOWED_NOMINATED_HEADERS) { + ENVOY_LOG_MISC(trace, "Too many nominated headers in request"); + return false; + } + + // Split the connection header and evaluate each nominated header + for (const auto& token : connection_header_tokens) { + + const auto token_sv = StringUtil::trim(token); + + // Build the LowerCaseString for header lookup + const LowerCaseString lcs_header_to_remove{std::string(token_sv)}; + + // If the Connection token value is not a nominated header, ignore it here since + // the connection header is removed elsewhere when the H1 request is upgraded to H2 + if ((lcs_header_to_remove.get() == cv.Close) || + (lcs_header_to_remove.get() == cv.Http2Settings) || + (lcs_header_to_remove.get() == cv.KeepAlive) || + (lcs_header_to_remove.get() == cv.Upgrade)) { + continue; + } + + // By default we will remove any nominated headers + bool keep_header = false; + + // Determine whether the nominated header contains invalid values + HeaderEntry* nominated_header = NULL; + + if (lcs_header_to_remove == Http::Headers::get().Connection) { + // Remove the connection header from the nominated tokens if it's self nominated + // The connection header itself is *not removed* + ENVOY_LOG_MISC(trace, "Skipping self nominated header [{}]", token_sv); + keep_header = true; + headers_to_remove.emplace(token_sv); + + } else if ((lcs_header_to_remove == Http::Headers::get().ForwardedFor) || + (lcs_header_to_remove == Http::Headers::get().ForwardedHost) || + (lcs_header_to_remove == Http::Headers::get().ForwardedProto) || + !token_sv.find(':')) { + + // An attacker could nominate an X-Forwarded* header, and its removal may mask + // the origin of the incoming request and potentially alter its handling. + // Additionally, pseudo headers should never be nominated. In both cases, we + // should fail the request. + // See: https://nathandavison.com/blog/abusing-http-hop-by-hop-request-headers + + ENVOY_LOG_MISC(trace, "Invalid nomination of {} header", token_sv); + return false; + } else { + // Examine the value of all other nominated headers + nominated_header = headers.get(lcs_header_to_remove); + } + + if (nominated_header) { + auto nominated_header_value_sv = nominated_header->value().getStringView(); + + const bool is_te_header = (lcs_header_to_remove == Http::Headers::get().TE); + + // reject the request if the TE header is too large + if (is_te_header && (nominated_header_value_sv.size() >= MAX_ALLOWED_TE_VALUE_SIZE)) { + ENVOY_LOG_MISC(trace, "TE header contains a value that exceeds the allowable length"); + return false; + } + + if (is_te_header) { + for (const auto& header_value : + StringUtil::splitToken(nominated_header_value_sv, ",", false)) { + + const absl::string_view header_sv = StringUtil::trim(header_value); + + // If trailers exist in the TE value tokens, keep the header, removing any other values + // that may exist + if (StringUtil::CaseInsensitiveCompare()(header_sv, + Http::Headers::get().TEValues.Trailers)) { + keep_header = true; + break; + } + } + + if (keep_header) { + nominated_header->value().setCopy(Http::Headers::get().TEValues.Trailers.data(), + Http::Headers::get().TEValues.Trailers.size()); + } + } + } + + if (!keep_header) { + ENVOY_LOG_MISC(trace, "Removing nominated header [{}]", token_sv); + headers.remove(lcs_header_to_remove); + headers_to_remove.emplace(token_sv); + } + } + + // Lastly remove extra nominated headers from the Connection header + if (!headers_to_remove.empty()) { + const std::string new_value = StringUtil::removeTokens(connection_header_value.getStringView(), + ",", headers_to_remove, ","); + + if (new_value.empty()) { + headers.removeConnection(); + } else { + headers.Connection()->value(new_value); + } + } + + return true; +} + const std::string& Utility::getProtocolString(const Protocol protocol) { switch (protocol) { case Protocol::Http10: @@ -568,8 +692,8 @@ std::string Utility::PercentEncoding::encode(absl::string_view value) { // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#responses. // // We do checking for each char in the string. If the current char is included in the defined - // escaping characters, we jump to "the slow path" (append the char [encoded or not encoded] to - // the returned string one by one) started from the current index. + // escaping characters, we jump to "the slow path" (append the char [encoded or not encoded] + // to the returned string one by one) started from the current index. if (ch < ' ' || ch >= '~' || ch == '%') { return PercentEncoding::encode(value, i); } diff --git a/source/common/http/utility.h b/source/common/http/utility.h index fef7b08e11b6..a46bd93a2b64 100644 --- a/source/common/http/utility.h +++ b/source/common/http/utility.h @@ -231,6 +231,15 @@ struct GetLastAddressFromXffInfo { GetLastAddressFromXffInfo getLastAddressFromXFF(const Http::HeaderMap& request_headers, uint32_t num_to_skip = 0); +/** + * Remove any headers nominated by the Connection header + * Sanitize the TE header if it contains unsupported values + * + * @param headers the client request headers + * @return whether the headers were sanitized successfully + */ +bool sanitizeConnectionHeader(Http::HeaderMap& headers); + /** * Get the string for the given http protocol. * @param protocol for which to return the string representation. diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 021da9af487d..5b5e410d865a 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -29,6 +29,7 @@ constexpr const char* runtime_features[] = { "envoy.reloadable_features.buffer_filter_populate_content_length", "envoy.reloadable_features.trusted_forwarded_proto", "envoy.reloadable_features.outlier_detection_support_for_grpc_status", + "envoy.reloadable_features.connection_header_sanitization", }; // This is a list of configuration fields which are disallowed by default in Envoy diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index 67585504416e..beddb6cce7c9 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -754,6 +754,280 @@ TEST(HttpUtility, GetMergedPerFilterConfig) { EXPECT_EQ(2, merged_cfg.value().state_); } +// Validates TE header is stripped if it contains an unsupported value +// Also validate the behavior if a nominated header does not exist +TEST(HttpUtility, TestTeHeaderGzipTrailersSanitized) { + TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, mike, sam, will, close"}, + {"te", "gzip, trailers"}, + {"sam", "bar"}, + {"will", "baz"}, + }; + + // Expect that the set of headers is valid and can be sanitized + EXPECT_TRUE(Utility::sanitizeConnectionHeader(request_headers)); + + Http::TestHeaderMapImpl sanitized_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te,close"}, + {"te", "trailers"}, + }; + EXPECT_EQ(sanitized_headers, request_headers); +} + +// Validates that if the connection header is nominated, the +// true connection header is not removed +TEST(HttpUtility, TestNominatedConnectionHeader) { + TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, mike, sam, will, connection, close"}, + {"te", "gzip"}, + {"sam", "bar"}, + {"will", "baz"}, + }; + EXPECT_TRUE(Utility::sanitizeConnectionHeader(request_headers)); + + TestHeaderMapImpl sanitized_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "close"}, + }; + EXPECT_EQ(sanitized_headers, request_headers); +} + +// Validate that if the connection header is nominated, we +// sanitize correctly preserving other nominated headers with +// supported values +TEST(HttpUtility, TestNominatedConnectionHeader2) { + Http::TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, mike, sam, will, connection, close"}, + {"te", "trailers"}, + {"sam", "bar"}, + {"will", "baz"}, + }; + EXPECT_TRUE(Utility::sanitizeConnectionHeader(request_headers)); + + Http::TestHeaderMapImpl sanitized_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te,close"}, + {"te", "trailers"}, + }; + EXPECT_EQ(sanitized_headers, request_headers); +} + +// Validate that connection is rejected if pseudo headers are nominated +// This includes an extra comma to ensure that the resulting +// header is still correct +TEST(HttpUtility, TestNominatedPseudoHeader) { + Http::TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, :path,, :method, :authority, connection, close"}, + {"te", "trailers"}, + }; + + // Headers remain unchanged since there are nominated pseudo headers + Http::TestHeaderMapImpl sanitized_headers(request_headers); + + EXPECT_FALSE(Utility::sanitizeConnectionHeader(request_headers)); + EXPECT_EQ(sanitized_headers, request_headers); +} + +// Validate that we can sanitize the headers when splitting +// the Connection header results in empty tokens +TEST(HttpUtility, TestSanitizeEmptyTokensFromHeaders) { + Http::TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, foo,, bar, close"}, + {"te", "trailers"}, + {"foo", "monday"}, + {"bar", "friday"}, + }; + EXPECT_TRUE(Utility::sanitizeConnectionHeader(request_headers)); + + Http::TestHeaderMapImpl sanitized_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te,close"}, + {"te", "trailers"}, + }; + EXPECT_EQ(sanitized_headers, request_headers); +} + +// Validate that we fail the request if there are too many +// nominated headers +TEST(HttpUtility, TestTooManyNominatedHeaders) { + Http::TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, connection, close, seahawks, niners, chargers, rams, raiders, " + "cardinals, eagles, giants, ravens"}, + {"te", "trailers"}, + }; + + // Headers remain unchanged because there are too many nominated headers + Http::TestHeaderMapImpl sanitized_headers(request_headers); + + EXPECT_FALSE(Utility::sanitizeConnectionHeader(request_headers)); + EXPECT_EQ(sanitized_headers, request_headers); +} + +TEST(HttpUtility, TestRejectNominatedXForwardedFor) { + Http::TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, x-forwarded-for"}, + {"te", "trailers"}, + }; + + // Headers remain unchanged due to nominated X-Forwarded* header + Http::TestHeaderMapImpl sanitized_headers(request_headers); + + EXPECT_FALSE(Utility::sanitizeConnectionHeader(request_headers)); + EXPECT_EQ(sanitized_headers, request_headers); +} + +TEST(HttpUtility, TestRejectNominatedXForwardedHost) { + Http::TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, x-forwarded-host"}, + {"te", "trailers"}, + }; + + // Headers remain unchanged due to nominated X-Forwarded* header + Http::TestHeaderMapImpl sanitized_headers(request_headers); + + EXPECT_FALSE(Utility::sanitizeConnectionHeader(request_headers)); + EXPECT_EQ(sanitized_headers, request_headers); +} + +TEST(HttpUtility, TestRejectNominatedXForwardedProto) { + Http::TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, x-forwarded-proto"}, + {"te", "TrAiLeRs"}, + }; + + // Headers are not sanitized due to nominated X-Forwarded* header + EXPECT_FALSE(Utility::sanitizeConnectionHeader(request_headers)); + + Http::TestHeaderMapImpl sanitized_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, x-forwarded-proto"}, + {"te", "trailers"}, + }; + EXPECT_EQ(sanitized_headers, request_headers); +} + +TEST(HttpUtility, TestRejectTrailersSubString) { + Http::TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, close"}, + {"te", "SemisWithTripleTrailersAreAthing"}, + }; + EXPECT_TRUE(Utility::sanitizeConnectionHeader(request_headers)); + + Http::TestHeaderMapImpl sanitized_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "close"}, + }; + EXPECT_EQ(sanitized_headers, request_headers); +} + +TEST(HttpUtility, TestRejectTeHeaderTooLong) { + Http::TestHeaderMapImpl request_headers = { + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, close"}, + {"te", "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef" + "1234567890abcdef"}, + }; + + // Headers remain unchanged because the TE value is too long + Http::TestHeaderMapImpl sanitized_headers(request_headers); + + EXPECT_FALSE(Utility::sanitizeConnectionHeader(request_headers)); + EXPECT_EQ(sanitized_headers, request_headers); +} + TEST(Url, ParsingFails) { Utility::Url url; EXPECT_FALSE(url.initialize("")); diff --git a/test/integration/header_integration_test.cc b/test/integration/header_integration_test.cc index a88a35a2e135..13bf87cd430a 100644 --- a/test/integration/header_integration_test.cc +++ b/test/integration/header_integration_test.cc @@ -4,6 +4,7 @@ #include "common/config/metadata.h" #include "common/config/resources.h" +#include "common/http/exception.h" #include "common/protobuf/protobuf.h" #include "test/integration/http_integration.h" @@ -1067,4 +1068,73 @@ TEST_P(HeaderIntegrationTest, TestPathAndRouteOnNormalizedPath) { {":status", "200"}, }); } + +// Validates TE header is forwarded if it contains a supported value +TEST_P(HeaderIntegrationTest, TestTeHeaderPassthrough) { + initializeFilter(HeaderMode::Append, false); + performRequest( + Http::TestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, close"}, + {"te", "trailers"}, + }, + Http::TestHeaderMapImpl{ + {":authority", "no-headers.com"}, + {":path", "/"}, + {":method", "GET"}, + {"x-request-foo", "downstram"}, + {"te", "trailers"}, + }, + Http::TestHeaderMapImpl{ + {"server", "envoy"}, + {"content-length", "0"}, + {":status", "200"}, + {"x-return-foo", "upstream"}, + }, + Http::TestHeaderMapImpl{ + {"server", "envoy"}, + {"x-return-foo", "upstream"}, + {":status", "200"}, + }); +} + +// Validates TE header is stripped if it contains an unsupported value +TEST_P(HeaderIntegrationTest, TestTeHeaderSanitized) { + initializeFilter(HeaderMode::Append, false); + performRequest( + Http::TestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "no-headers.com"}, + {"x-request-foo", "downstram"}, + {"connection", "te, mike, sam, will, close"}, + {"te", "gzip"}, + {"mike", "foo"}, + {"sam", "bar"}, + {"will", "baz"}, + }, + Http::TestHeaderMapImpl{ + {":authority", "no-headers.com"}, + {":path", "/"}, + {":method", "GET"}, + {"x-request-foo", "downstram"}, + }, + Http::TestHeaderMapImpl{ + {"server", "envoy"}, + {"content-length", "0"}, + {":status", "200"}, + {"x-return-foo", "upstream"}, + }, + Http::TestHeaderMapImpl{ + {"server", "envoy"}, + {"x-return-foo", "upstream"}, + {":status", "200"}, + {"connection", "close"}, + }); +} } // namespace Envoy From 857a96dbc144ee826010d6a6293f3accf95e8ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Tue, 19 Nov 2019 13:11:02 -0300 Subject: [PATCH 04/29] router: add Location header only when required (#9065) Change the router filter to only add the location header to 201 and 3xx responses. Tests: new UT Fixes: #9063 Signed-off-by: Raul Gutierrez Segales --- docs/root/intro/version_history.rst | 1 + source/common/router/router.cc | 6 +++- test/common/router/router_test.cc | 46 +++++++++++++++++++++++++++-- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 20813ba2ca2a..3ecee6399aca 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -16,6 +16,7 @@ Version history * logger: added :ref:`--log-format-escaped ` command line option to escape newline characters in application logs. * redis: performance improvement for larger split commands by avoiding string copies. * router: added support for REQ(header-name) :ref:`header formatter `. +* router: skip the Location header when the response code is not a 201 or a 3xx. * server: fixed a bug in config validation for configs with runtime layers * tcp_proxy: added :ref:`ClusterWeight.metadata_match` * tcp_proxy: added :ref:`hash_policy` diff --git a/source/common/router/router.cc b/source/common/router/router.cc index 5799add7dccd..648a5d491135 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -391,7 +391,11 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::HeaderMap& headers, bool e [this, direct_response, &request_headers = headers](Http::HeaderMap& response_headers) -> void { const auto new_path = direct_response->newPath(request_headers); - if (!new_path.empty()) { + // See https://tools.ietf.org/html/rfc7231#section-7.1.2. + const auto add_location = + direct_response->responseCode() == Http::Code::Created || + Http::CodeUtility::is3xx(enumToInt(direct_response->responseCode())); + if (!new_path.empty() && add_location) { response_headers.addReferenceKey(Http::Headers::get().Location, new_path); } direct_response->finalizeResponseHeaders(response_headers, callbacks_->streamInfo()); diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index c8d58303b25c..8676fc3dd6ae 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -3330,7 +3330,7 @@ TEST_F(RouterTest, Redirect) { EXPECT_CALL(direct_response, newPath(_)).WillOnce(Return("hello")); EXPECT_CALL(direct_response, routeName()).WillOnce(ReturnRef(route_name)); EXPECT_CALL(direct_response, rewritePathHeader(_, _)); - EXPECT_CALL(direct_response, responseCode()).WillOnce(Return(Http::Code::MovedPermanently)); + EXPECT_CALL(direct_response, responseCode()).WillRepeatedly(Return(Http::Code::MovedPermanently)); EXPECT_CALL(direct_response, responseBody()).WillOnce(ReturnRef(EMPTY_STRING)); EXPECT_CALL(direct_response, finalizeResponseHeaders(_, _)); EXPECT_CALL(*callbacks_.route_, directResponseEntry()).WillRepeatedly(Return(&direct_response)); @@ -3351,7 +3351,7 @@ TEST_F(RouterTest, RedirectFound) { EXPECT_CALL(direct_response, newPath(_)).WillOnce(Return("hello")); EXPECT_CALL(direct_response, routeName()).WillOnce(ReturnRef(route_name)); EXPECT_CALL(direct_response, rewritePathHeader(_, _)); - EXPECT_CALL(direct_response, responseCode()).WillOnce(Return(Http::Code::Found)); + EXPECT_CALL(direct_response, responseCode()).WillRepeatedly(Return(Http::Code::Found)); EXPECT_CALL(direct_response, responseBody()).WillOnce(ReturnRef(EMPTY_STRING)); EXPECT_CALL(direct_response, finalizeResponseHeaders(_, _)); EXPECT_CALL(*callbacks_.route_, directResponseEntry()).WillRepeatedly(Return(&direct_response)); @@ -3408,6 +3408,48 @@ TEST_F(RouterTest, DirectResponseWithBody) { EXPECT_EQ(1UL, config_.stats_.rq_direct_response_.value()); } +TEST_F(RouterTest, DirectResponseWithLocation) { + NiceMock direct_response; + std::string route_name("route-test-name"); + EXPECT_CALL(direct_response, newPath(_)).WillOnce(Return("http://host/")); + EXPECT_CALL(direct_response, routeName()).WillOnce(ReturnRef(route_name)); + EXPECT_CALL(direct_response, responseCode()).WillRepeatedly(Return(Http::Code::Created)); + EXPECT_CALL(direct_response, responseBody()).WillRepeatedly(ReturnRef(EMPTY_STRING)); + EXPECT_CALL(*callbacks_.route_, directResponseEntry()).WillRepeatedly(Return(&direct_response)); + absl::string_view route_name_view(route_name); + EXPECT_CALL(callbacks_.stream_info_, setRouteName(route_name_view)); + + Http::TestHeaderMapImpl response_headers{{":status", "201"}, {"location", "http://host/"}}; + EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), true)); + EXPECT_CALL(span_, injectContext(_)).Times(0); + Http::TestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + EXPECT_TRUE(verifyHostUpstreamStats(0, 0)); + EXPECT_EQ(1UL, config_.stats_.rq_direct_response_.value()); +} + +TEST_F(RouterTest, DirectResponseWithoutLocation) { + NiceMock direct_response; + std::string route_name("route-test-name"); + EXPECT_CALL(direct_response, newPath(_)).WillOnce(Return("http://host/")); + EXPECT_CALL(direct_response, routeName()).WillOnce(ReturnRef(route_name)); + EXPECT_CALL(direct_response, responseCode()).WillRepeatedly(Return(Http::Code::OK)); + EXPECT_CALL(direct_response, responseBody()).WillRepeatedly(ReturnRef(EMPTY_STRING)); + EXPECT_CALL(*callbacks_.route_, directResponseEntry()).WillRepeatedly(Return(&direct_response)); + absl::string_view route_name_view(route_name); + EXPECT_CALL(callbacks_.stream_info_, setRouteName(route_name_view)); + + Http::TestHeaderMapImpl response_headers{{":status", "200"}}; + EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), true)); + EXPECT_CALL(span_, injectContext(_)).Times(0); + Http::TestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + EXPECT_TRUE(verifyHostUpstreamStats(0, 0)); + EXPECT_EQ(1UL, config_.stats_.rq_direct_response_.value()); +} + TEST_F(RouterTest, UpstreamSSLConnection) { NiceMock encoder; Http::StreamDecoder* response_decoder = nullptr; From c40b96bb0ff869c8195a4e2ca9f8fa12a7380372 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 19 Nov 2019 09:42:00 -0800 Subject: [PATCH 05/29] yaml: use protobuf to parse float/double (#9052) Description: Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=18977 Risk Level: Low Testing: unit test, fuzz Docs Changes: N/A Signed-off-by: Lizan Zhou --- source/common/protobuf/utility.cc | 10 +++------- ...-testcase-minimized-json_fuzz_test-5724109283786752 | 1 + test/common/protobuf/utility_test.cc | 10 ++++++---- test/common/router/config_impl_test.cc | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 test/common/json/json_corpus/clusterfuzz-testcase-minimized-json_fuzz_test-5724109283786752 diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index afcf72844261..0fa1396433cd 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -64,17 +64,13 @@ ProtobufWkt::Value parseYamlNode(const YAML::Node& node) { value.set_number_value(int_value); } else { // Proto3 JSON mapping allows use string for integer, this still has to be converted from - // int_value to support 0x and 0o literals. + // int_value to support hexadecimal and octal literals. value.set_string_value(std::to_string(int_value)); } break; } - double double_value; - if (YAML::convert::decode(node, double_value)) { - value.set_number_value(double_value); - break; - } - // Otherwise, fall back on string. + // Fall back on string, including float/double case. When protobuf parse the JSON into a message + // it will convert based on the type in the message definition. value.set_string_value(node.as()); break; } diff --git a/test/common/json/json_corpus/clusterfuzz-testcase-minimized-json_fuzz_test-5724109283786752 b/test/common/json/json_corpus/clusterfuzz-testcase-minimized-json_fuzz_test-5724109283786752 new file mode 100644 index 000000000000..1873bd50d57f --- /dev/null +++ b/test/common/json/json_corpus/clusterfuzz-testcase-minimized-json_fuzz_test-5724109283786752 @@ -0,0 +1 @@ +{s:"08955690052"} diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index d79f83743086..7d782df391cb 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -357,6 +357,12 @@ TEST_F(ProtobufUtilityTest, ValueUtilHash) { EXPECT_NE(ValueUtil::hash(v), 0); } +TEST_F(ProtobufUtilityTest, MessageUtilLoadYamlDouble) { + ProtobufWkt::DoubleValue v; + MessageUtil::loadFromYaml("value: 1.0", v, ProtobufMessage::getNullValidationVisitor()); + EXPECT_DOUBLE_EQ(1.0, v.value()); +} + TEST_F(ProtobufUtilityTest, ValueUtilLoadFromYamlScalar) { EXPECT_EQ(ValueUtil::loadFromYaml("null").ShortDebugString(), "null_value: NULL_VALUE"); EXPECT_EQ(ValueUtil::loadFromYaml("true").ShortDebugString(), "bool_value: true"); @@ -365,10 +371,6 @@ TEST_F(ProtobufUtilityTest, ValueUtilLoadFromYamlScalar) { "string_value: \"9223372036854775807\""); EXPECT_EQ(ValueUtil::loadFromYaml("\"foo\"").ShortDebugString(), "string_value: \"foo\""); EXPECT_EQ(ValueUtil::loadFromYaml("foo").ShortDebugString(), "string_value: \"foo\""); - { - ProtobufWkt::Value v = ValueUtil::loadFromYaml("1.0"); - EXPECT_DOUBLE_EQ(1.0, v.number_value()); - } } TEST_F(ProtobufUtilityTest, ValueUtilLoadFromYamlObject) { diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 578d2a621a7b..1cdb2ae411cd 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -3179,7 +3179,7 @@ name: HedgeVirtualHostLevel hedge_policy: {hedge_on_per_try_timeout: true} - match: {prefix: /bar} route: - hedge_policy: {additional_request_chance: {numerator: 30.0, denominator: HUNDRED}} + hedge_policy: {additional_request_chance: {numerator: 30, denominator: HUNDRED}} cluster: www - match: {prefix: /} route: {cluster: www} From 74a5a07d25c02f44aff2a154a6535c25ff23d292 Mon Sep 17 00:00:00 2001 From: Marco Magdy Date: Tue, 19 Nov 2019 11:58:46 -0800 Subject: [PATCH 06/29] AWS X-Ray Localized Sampling (1/2) (#8972) Localized sampling strategy for X-Ray (1/2). This creates the spans but does not (yet) send them to the daemon. Risk Level: Low Testing: Unit tests Signed-off-by: Marco Magdy --- source/extensions/tracers/xray/BUILD | 7 + source/extensions/tracers/xray/config.cc | 4 +- .../tracers/xray/localized_sampling.cc | 207 ++++++ .../tracers/xray/localized_sampling.h | 175 +++++ source/extensions/tracers/xray/reservoir.h | 73 +++ .../tracers/xray/sampling_strategy.h | 34 +- source/extensions/tracers/xray/tracer.cc | 61 ++ source/extensions/tracers/xray/tracer.h | 140 +++- .../tracers/xray/xray_configuration.h | 25 +- .../tracers/xray/xray_tracer_impl.cc | 94 ++- .../tracers/xray/xray_tracer_impl.h | 13 +- test/extensions/tracers/xray/BUILD | 11 + .../tracers/xray/localized_sampling_test.cc | 612 ++++++++++++++++++ .../tracers/xray/xray_tracer_impl_test.cc | 91 +++ 14 files changed, 1496 insertions(+), 51 deletions(-) create mode 100644 source/extensions/tracers/xray/localized_sampling.cc create mode 100644 source/extensions/tracers/xray/localized_sampling.h create mode 100644 source/extensions/tracers/xray/reservoir.h create mode 100644 source/extensions/tracers/xray/tracer.cc create mode 100644 test/extensions/tracers/xray/localized_sampling_test.cc create mode 100644 test/extensions/tracers/xray/xray_tracer_impl_test.cc diff --git a/source/extensions/tracers/xray/BUILD b/source/extensions/tracers/xray/BUILD index 3440de74b856..2e842f07b9f4 100644 --- a/source/extensions/tracers/xray/BUILD +++ b/source/extensions/tracers/xray/BUILD @@ -14,23 +14,30 @@ envoy_package() envoy_cc_library( name = "xray_lib", srcs = [ + "localized_sampling.cc", + "tracer.cc", "util.cc", "xray_tracer_impl.cc", ], hdrs = [ + "localized_sampling.h", + "reservoir.h", "sampling_strategy.h", "tracer.h", "util.h", "xray_configuration.h", "xray_tracer_impl.h", ], + external_deps = [], deps = [ "//include/envoy/common:time_interface", "//include/envoy/server:instance_interface", "//include/envoy/tracing:http_tracer_interface", + "//source/common/common:hex_lib", "//source/common/common:macros", "//source/common/http:header_map_lib", "//source/common/json:json_loader_lib", + "//source/common/protobuf:utility_lib", "//source/common/tracing:http_tracer_lib", ], ) diff --git a/source/extensions/tracers/xray/config.cc b/source/extensions/tracers/xray/config.cc index e19705bd9b8a..97209173428e 100644 --- a/source/extensions/tracers/xray/config.cc +++ b/source/extensions/tracers/xray/config.cc @@ -29,8 +29,8 @@ XRayTracerFactory::createHttpTracerTyped(const envoy::config::trace::v2::XRayCon ENVOY_LOG(error, "Failed to read sampling rules manifest because of {}.", e.what()); } - XRayConfiguration xconfig(proto_config.daemon_endpoint(), proto_config.segment_name(), - sampling_rules_json); + XRayConfiguration xconfig{proto_config.daemon_endpoint(), proto_config.segment_name(), + sampling_rules_json}; auto xray_driver = std::make_unique(xconfig, server); return std::make_unique(std::move(xray_driver), server.localInfo()); diff --git a/source/extensions/tracers/xray/localized_sampling.cc b/source/extensions/tracers/xray/localized_sampling.cc new file mode 100644 index 000000000000..bb8d509c3deb --- /dev/null +++ b/source/extensions/tracers/xray/localized_sampling.cc @@ -0,0 +1,207 @@ +#include "extensions/tracers/xray/localized_sampling.h" + +#include "common/http/exception.h" +#include "common/protobuf/utility.h" + +#include "extensions/tracers/xray/util.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace XRay { + +constexpr static double DefaultRate = 0.5; +constexpr static int DefaultFixedTarget = 1; +constexpr static int SamplingFileVersion = 2; +constexpr static char VersionJsonKey[] = "version"; +constexpr static char DefaultRuleJsonKey[] = "default"; +constexpr static char FixedTargetJsonKey[] = "fixed_target"; +constexpr static char RateJsonKey[] = "rate"; +constexpr static char CustomRulesJsonKey[] = "rules"; +constexpr static char HostJsonKey[] = "host"; +constexpr static char HttpMethodJsonKey[] = "http_method"; +constexpr static char UrlPathJsonKey[] = "url_path"; + +namespace { +void fail(absl::string_view msg) { + auto& logger = Logger::Registry::getLog(Logger::Id::tracing); + ENVOY_LOG_TO_LOGGER(logger, error, "Failed to parse sampling rules - {}", msg); +} + +bool is_valid_rate(double n) { return n >= 0 && n <= 1.0; } +bool is_valid_fixed_target(double n) { return n >= 0 && static_cast(n) == n; } +} // namespace + +static bool validateRule(const ProtobufWkt::Struct& rule) { + using ProtobufWkt::Value; + + const auto host_it = rule.fields().find(HostJsonKey); + if (host_it != rule.fields().end() && + host_it->second.kind_case() != Value::KindCase::kStringValue) { + fail("host must be a string"); + return false; + } + + const auto http_method_it = rule.fields().find(HttpMethodJsonKey); + if (http_method_it != rule.fields().end() && + http_method_it->second.kind_case() != Value::KindCase::kStringValue) { + fail("HTTP method must be a string"); + return false; + } + + const auto url_path_it = rule.fields().find(UrlPathJsonKey); + if (url_path_it != rule.fields().end() && + url_path_it->second.kind_case() != Value::KindCase::kStringValue) { + fail("URL path must be a string"); + return false; + } + + const auto fixed_target_it = rule.fields().find(FixedTargetJsonKey); + if (fixed_target_it == rule.fields().end() || + fixed_target_it->second.kind_case() != Value::KindCase::kNumberValue || + !is_valid_fixed_target(fixed_target_it->second.number_value())) { + fail("fixed target is missing or not a valid positive integer"); + return false; + } + + const auto rate_it = rule.fields().find(RateJsonKey); + if (rate_it == rule.fields().end() || + rate_it->second.kind_case() != Value::KindCase::kNumberValue || + !is_valid_rate(rate_it->second.number_value())) { + fail("rate is missing or not a valid positive floating number"); + return false; + } + return true; +} + +LocalizedSamplingRule LocalizedSamplingRule::createDefault() { + return LocalizedSamplingRule(DefaultFixedTarget, DefaultRate); +} + +bool LocalizedSamplingRule::appliesTo(const SamplingRequest& request) const { + return (request.host_.empty() || wildcardMatch(host_, request.host_)) && + (request.http_method_.empty() || wildcardMatch(http_method_, request.http_method_)) && + (request.http_url_.empty() || wildcardMatch(url_path_, request.http_url_)); +} + +LocalizedSamplingManifest::LocalizedSamplingManifest(const std::string& rule_json) + : default_rule_(LocalizedSamplingRule::createDefault()) { + if (rule_json.empty()) { + return; + } + + ProtobufWkt::Struct document; + try { + MessageUtil::loadFromJson(rule_json, document); + } catch (EnvoyException& e) { + fail("invalid JSON format"); + return; + } + + const auto version_it = document.fields().find(VersionJsonKey); + if (version_it == document.fields().end()) { + fail("missing version number"); + return; + } + + if (version_it->second.kind_case() != ProtobufWkt::Value::KindCase::kNumberValue || + version_it->second.number_value() != SamplingFileVersion) { + fail("wrong version number"); + return; + } + + const auto default_rule_it = document.fields().find(DefaultRuleJsonKey); + if (default_rule_it == document.fields().end() || + default_rule_it->second.kind_case() != ProtobufWkt::Value::KindCase::kStructValue) { + fail("missing default rule"); + return; + } + + // extract default rule members + auto& default_rule_object = default_rule_it->second.struct_value(); + if (!validateRule(default_rule_object)) { + return; + } + + default_rule_.setRate(default_rule_object.fields().find(RateJsonKey)->second.number_value()); + default_rule_.setFixedTarget(static_cast( + default_rule_object.fields().find(FixedTargetJsonKey)->second.number_value())); + + const auto custom_rules_it = document.fields().find(CustomRulesJsonKey); + if (custom_rules_it == document.fields().end()) { + return; + } + + if (custom_rules_it->second.kind_case() != ProtobufWkt::Value::KindCase::kListValue) { + fail("rules must be JSON array"); + return; + } + + for (auto& el : custom_rules_it->second.list_value().values()) { + if (el.kind_case() != ProtobufWkt::Value::KindCase::kStructValue) { + fail("rules array must be objects"); + return; + } + + auto& rule_json = el.struct_value(); + if (!validateRule(rule_json)) { + return; + } + + LocalizedSamplingRule rule = LocalizedSamplingRule::createDefault(); + const auto host_it = rule_json.fields().find(HostJsonKey); + if (host_it != rule_json.fields().end()) { + rule.setHost(host_it->second.string_value()); + } + + const auto http_method_it = rule_json.fields().find(HttpMethodJsonKey); + if (http_method_it != rule_json.fields().end()) { + rule.setHttpMethod(http_method_it->second.string_value()); + } + + const auto url_path_it = rule_json.fields().find(UrlPathJsonKey); + if (url_path_it != rule_json.fields().end()) { + rule.setUrlPath(url_path_it->second.string_value()); + } + + // rate and fixed_target must exist because we validated this rule + rule.setRate(rule_json.fields().find(RateJsonKey)->second.number_value()); + rule.setFixedTarget( + static_cast(rule_json.fields().find(FixedTargetJsonKey)->second.number_value())); + + custom_rules_.push_back(std::move(rule)); + } +} + +bool LocalizedSamplingStrategy::shouldTrace(const SamplingRequest& sampling_request) { + if (!custom_manifest_.hasCustomRules()) { + return shouldTrace(default_manifest_.defaultRule()); + } + + for (auto&& rule : custom_manifest_.customRules()) { + if (rule.appliesTo(sampling_request)) { + return shouldTrace(rule); + } + } + return shouldTrace(custom_manifest_.defaultRule()); +} + +bool LocalizedSamplingStrategy::shouldTrace(LocalizedSamplingRule& rule) { + const auto now = time_source_.monotonicTime(); + if (rule.reservoir().take(now)) { + return true; + } + + // rule.rate() is a rational number between 0 and 1 + auto toss = random() % 100; + if (toss < (100 * rule.rate())) { + return true; + } + + return false; +} + +} // namespace XRay +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/xray/localized_sampling.h b/source/extensions/tracers/xray/localized_sampling.h new file mode 100644 index 000000000000..709ec144a32b --- /dev/null +++ b/source/extensions/tracers/xray/localized_sampling.h @@ -0,0 +1,175 @@ +#pragma once + +#include +#include +#include + +#include "envoy/common/time.h" +#include "envoy/runtime/runtime.h" + +#include "common/common/logger.h" + +#include "extensions/tracers/xray/reservoir.h" +#include "extensions/tracers/xray/sampling_strategy.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace XRay { + +/** + * This class encompasses the algorithm used when deciding whether to sample a given request. + * The rule contains wildcard strings for matching a request based on its hostname, HTTP method or + * URL path. A request must match on all 3 parts before this rule is applied. + * If the rule applies, then |fixed_target| determines how many requests to sample per second. + * While, rate determines the percentage of requests to sample after that within the same second. + * + * By default, this tracer records the first request each second, and five percent of + * any additional requests. + */ +class LocalizedSamplingRule { +public: + /** + * Creates a default sampling rule that has the default |fixed_target| and default |rate| set. + */ + static LocalizedSamplingRule createDefault(); + + LocalizedSamplingRule(uint32_t fixed_target, double rate) + : fixed_target_(fixed_target), rate_(rate), reservoir_(fixed_target_) {} + + /** + * Determines whether Hostname, HTTP method and URL path match the given request. + */ + bool appliesTo(const SamplingRequest& request) const; + + /** + * Set the hostname to match against. + * This value can contain wildcard characters such as '*' or '?'. + */ + void setHost(absl::string_view host) { host_ = std::string(host); } + + /** + * Set the HTTP method to match against. + * This value can contain wildcard characters such as '*' or '?'. + */ + void setHttpMethod(absl::string_view http_method) { http_method_ = std::string(http_method); } + + /** + * Set the URL path to match against. + * This value can contain wildcard characters such as '*' or '?'. + */ + void setUrlPath(absl::string_view url_path) { url_path_ = std::string(url_path); } + + /** + * Set the minimum number of requests to sample per second. + */ + void setFixedTarget(uint32_t fixed_target) { + fixed_target_ = fixed_target; + reservoir_ = Reservoir(fixed_target); + } + + /** + * Set the percentage of requests to sample _after_ sampling |fixed_target| requests per second. + */ + void setRate(double rate) { rate_ = rate; } + + const std::string& host() const { return host_; } + const std::string& httpMethod() const { return http_method_; } + const std::string& urlPath() const { return url_path_; } + uint32_t fixedTarget() const { return fixed_target_; } + double rate() const { return rate_; } + const Reservoir& reservoir() const { return reservoir_; } + Reservoir& reservoir() { return reservoir_; } + +private: + std::string host_; + std::string http_method_; + std::string url_path_; + uint32_t fixed_target_; + double rate_; + Reservoir reservoir_; +}; + +/** + * The manifest represents the set of sampling rules (custom and default) used to match incoming + * requests. + */ +class LocalizedSamplingManifest { +public: + /** + * Create a default manifest. The default manifest is used when a custom manifest does not exist + * or failed to parse. The default manifest, will have an empty set of custom rules. + */ + static LocalizedSamplingManifest createDefault() { + return LocalizedSamplingManifest{LocalizedSamplingRule::createDefault()}; + } + + /** + * Create a manifest by de-serializing the input string as JSON representation of the sampling + * rules. + * @param sampling_rules_json JSON representation of X-Ray localized sampling rules. + */ + explicit LocalizedSamplingManifest(const std::string& sampling_rules_json); + + /** + * Create a manifest by assigning the argument rule as the default rule. The set of custom rules + * in this manifest will be empty. + * @param default_rule A localized sampling rule that will be assigned as the default rule. + */ + explicit LocalizedSamplingManifest(const LocalizedSamplingRule& default_rule) + : default_rule_(default_rule) {} + + /** + * @return default sampling rule + */ + LocalizedSamplingRule& defaultRule() { return default_rule_; } + + /** + * @return the user-defined sampling rules + */ + std::vector& customRules() { return custom_rules_; } + + /** + * @return true if the this manifest has a set of custom rules; otherwise false. + */ + bool hasCustomRules() const { return !custom_rules_.empty(); } + +private: + LocalizedSamplingRule default_rule_; + std::vector custom_rules_; +}; + +class LocalizedSamplingStrategy : public SamplingStrategy { +public: + LocalizedSamplingStrategy(const std::string& sampling_rules_json, Runtime::RandomGenerator& rng, + TimeSource& time_source) + : SamplingStrategy(rng), default_manifest_(LocalizedSamplingManifest::createDefault()), + custom_manifest_(sampling_rules_json), time_source_(time_source), + use_default_(!custom_manifest_.hasCustomRules()) {} + + /** + * Determines if an incoming request matches one of the sampling rules in the local manifests. + * If a match is found, then the request might be traced based on the sampling percentages etc. + * determined by the matching rule. + */ + bool shouldTrace(const SamplingRequest& sampling_request) override; + + /** + * Determines whether default rules are in effect. Mainly for unit testing purposes. + */ + bool usingDefaultManifest() const { return use_default_; } + +private: + bool shouldTrace(LocalizedSamplingRule& rule); + LocalizedSamplingManifest default_manifest_; + LocalizedSamplingManifest custom_manifest_; + TimeSource& time_source_; + bool use_default_; +}; + +} // namespace XRay +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/xray/reservoir.h b/source/extensions/tracers/xray/reservoir.h new file mode 100644 index 000000000000..fc8da633c471 --- /dev/null +++ b/source/extensions/tracers/xray/reservoir.h @@ -0,0 +1,73 @@ +#pragma once + +#include + +#include "envoy/common/time.h" + +#include "common/common/lock_guard.h" +#include "common/common/thread.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace XRay { + +/** + * Simple token-bucket algorithm that enables counting samples/traces used per second. + */ +class Reservoir { +public: + /** + * Creates a new reservoir that allows up to |traces_per_second| samples. + */ + explicit Reservoir(uint32_t traces_per_second) + : traces_per_second_(traces_per_second), used_(0) {} + + Reservoir(const Reservoir& other) + : traces_per_second_(other.traces_per_second_), used_(other.used_), + time_point_(other.time_point_) {} + + Reservoir& operator=(const Reservoir& other) { + if (this == &other) { + return *this; + } + traces_per_second_ = other.traces_per_second_; + used_ = other.used_; + time_point_ = other.time_point_; + return *this; + } + + /** + * Determines whether all samples have been used up for this particular second. + * Every second, this reservoir starts over with a full bucket. + * + * @param now Used to compare against the last recorded time to determine if it's still within the + * same second. + */ + bool take(Envoy::MonotonicTime now) { + Envoy::Thread::LockGuard lg(sync_); + const auto diff = now - time_point_; + if (diff > std::chrono::seconds(1)) { + used_ = 0; + time_point_ = now; + } + + if (used_ >= traces_per_second_) { + return false; + } + + ++used_; + return true; + } + +private: + uint32_t traces_per_second_; + uint32_t used_; + Envoy::MonotonicTime time_point_; + Envoy::Thread::MutexBasicLockable sync_; +}; + +} // namespace XRay +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/xray/sampling_strategy.h b/source/extensions/tracers/xray/sampling_strategy.h index 5fed69492e63..2cb488a2c33f 100644 --- a/source/extensions/tracers/xray/sampling_strategy.h +++ b/source/extensions/tracers/xray/sampling_strategy.h @@ -1,9 +1,9 @@ #pragma once #include -#include #include "envoy/common/pure.h" +#include "envoy/runtime/runtime.h" #include "common/common/macros.h" @@ -15,22 +15,9 @@ namespace Tracers { namespace XRay { struct SamplingRequest { - /** - * Creates a new SamplingRequest - * - * @param host_name The host name the request. - * @param http_method The http method of the request e.g. GET, POST, etc. - * @param http_url The path part of the URL of the request. - * @param service The name of the service (user specified) - * @param service_type The type of the service (user specified) - */ - SamplingRequest(absl::string_view host_name, absl::string_view http_method, - absl::string_view http_url) - : host_(host_name), http_method_(http_method), http_url_(http_url) {} - - const std::string host_; - const std::string http_method_; - const std::string http_url_; + std::string host_; + std::string http_method_; + std::string http_url_; }; /** @@ -38,19 +25,20 @@ struct SamplingRequest { */ class SamplingStrategy { public: - explicit SamplingStrategy(uint64_t rng_seed) : rng_(rng_seed) {} + explicit SamplingStrategy(Runtime::RandomGenerator& rng) : rng_(rng) {} virtual ~SamplingStrategy() = default; /** * sampleRequest determines if the given request should be traced or not. + * Implementation _must_ be thread-safe. */ - virtual bool sampleRequest(const SamplingRequest& sampling_request) { - UNREFERENCED_PARAMETER(sampling_request); // unused for now - return rng_() % 100 == 42; - } + virtual bool shouldTrace(const SamplingRequest& sampling_request) PURE; + +protected: + uint64_t random() const { return rng_.random(); } private: - std::mt19937 rng_; + Runtime::RandomGenerator& rng_; }; using SamplingStrategyPtr = std::unique_ptr; diff --git a/source/extensions/tracers/xray/tracer.cc b/source/extensions/tracers/xray/tracer.cc new file mode 100644 index 000000000000..c4aee2e5edb7 --- /dev/null +++ b/source/extensions/tracers/xray/tracer.cc @@ -0,0 +1,61 @@ +#include "extensions/tracers/xray/tracer.h" + +#include +#include +#include + +#include "envoy/http/header_map.h" + +#include "common/common/fmt.h" +#include "common/common/hex.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace XRay { + +void Span::finishSpan() { + end_time_ = time_source_.systemTime(); + // TODO(marcomagdy): serialize this span and send it to X-Ray Daemon +} + +void Span::injectContext(Http::HeaderMap& request_headers) { + const std::string xray_header_value = + fmt::format("root={};parent={};sampled=1", traceId(), parent_segment_id_); + + // Set the XRay header in to envoy header map. + request_headers.setReferenceKey(Http::LowerCaseString(XRayTraceHeader), xray_header_value); +} + +Tracing::SpanPtr Tracer::startSpan(const std::string& span_name, const std::string& operation_name, + Envoy::SystemTime start_time, + const absl::optional& xray_header) { + const auto ticks = time_source_.monotonicTime().time_since_epoch().count(); + auto span_ptr = std::make_unique(time_source_); + span_ptr->setId(ticks); + span_ptr->setName(span_name); + span_ptr->setOperation(operation_name); + span_ptr->setStartTime(start_time); + if (xray_header) { // there's a previous span that this span should be based-on + span_ptr->setParentId(xray_header->parent_id_); + span_ptr->setTraceId(xray_header->trace_id_); + switch (xray_header->sample_decision_) { + case SamplingDecision::Sampled: + span_ptr->setSampled(true); + break; + case SamplingDecision::NotSampled: + span_ptr->setSampled(false); + break; + default: + break; + } + } else { + span_ptr->setTraceId(Hex::uint64ToHex(ticks)); + } + return span_ptr; +} + +} // namespace XRay +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/xray/tracer.h b/source/extensions/tracers/xray/tracer.h index b201b858a558..1b3f429a87c2 100644 --- a/source/extensions/tracers/xray/tracer.h +++ b/source/extensions/tracers/xray/tracer.h @@ -1,12 +1,16 @@ #pragma once #include +#include +#include #include "envoy/common/time.h" #include "envoy/tracing/http_tracer.h" #include "extensions/tracers/xray/sampling_strategy.h" +#include "extensions/tracers/xray/xray_configuration.h" +#include "absl/container/flat_hash_map.h" #include "absl/strings/string_view.h" namespace Envoy { @@ -14,6 +18,134 @@ namespace Extensions { namespace Tracers { namespace XRay { +static const char XRayTraceHeader[] = "x-amzn-trace-id"; + +class Span : public Tracing::Span { +public: + /** + * Construct a new X-Ray Span. + */ + Span(TimeSource& time_source) : time_source_(time_source) {} + + /** + * Set the Span's trace ID. + */ + void setTraceId(absl::string_view trace_id) { trace_id_ = std::string(trace_id); }; + + /** + * Get the Span's trace ID. + */ + const std::string& traceId() const { return trace_id_; } + + /** + * Complete the current span, serialize it and send it to the X-Ray daemon. + */ + void finishSpan() override; + + /** + * Set the current operation name on the Span. + * This information will be included in the X-Ray span's metadata. + */ + void setOperation(absl::string_view operation) override { + operation_name_ = std::string(operation); + } + + /** + * Set the name of the Span. + */ + void setName(absl::string_view name) { name_ = std::string(name); } + + /** + * Add a key-value pair to either the Span's annotations or metadata. + * A whitelist of keys are added to the annotations, everything else is added to the metadata. + */ + void setTag(absl::string_view name, absl::string_view value) override { + UNREFERENCED_PARAMETER(name); + UNREFERENCED_PARAMETER(value); + } + + /** + * Set the ID of the parent segment. This is different from the Trace ID. + * The parent ID is used if the request originated from an instrumented application. + * For more information see: + * https://docs.aws.amazon.com/xray/latest/devguide/xray-concepts.html#xray-concepts-tracingheader + */ + void setParentId(absl::string_view parent_segment_id) { + parent_segment_id_ = std::string(parent_segment_id); + } + + /** + * Set the recording start time of the traced operation/request. + */ + void setStartTime(Envoy::SystemTime start_time) { start_time_ = start_time; } + + /** + * Set the recording end time of the traced operation/request. + */ + void setEndTime(Envoy::SystemTime end_time) { end_time_ = end_time; } + + /** + * Set the Span ID. + * This ID is used as the (sub)segment ID. + * A single Trace can have Multiple segments and each segment can have multiple sub-segments. + */ + void setId(uint64_t id) { id_ = id; } + + /** + * Not used by X-Ray. + */ + void setSampled(bool sampled) override { UNREFERENCED_PARAMETER(sampled); }; + + /** + * Not used by X-Ray. + */ + void injectContext(Http::HeaderMap& request_headers) override; + + /** + * Get the start time of this Span. + */ + Envoy::SystemTime startTime() const { return start_time_; } + + /** + * Get the end time of this Span. + */ + Envoy::SystemTime endTime() const { return end_time_; } + + /** + * Get this Span's ID. + */ + uint64_t Id() const { return id_; } + + /** + * Not used by X-Ray because the Spans are "logged" (serialized) to the X-Ray daemon. + */ + void log(Envoy::SystemTime, const std::string&) override {} + + /** + * Not supported by X-Ray. + */ + Tracing::SpanPtr spawnChild(const Tracing::Config&, const std::string&, + Envoy::SystemTime) override { + return nullptr; + } + +private: + void setParentSpan(Tracing::Span* parent) { parent_span_ = parent; } + + TimeSource& time_source_; + Envoy::SystemTime start_time_; + Envoy::SystemTime end_time_; + std::string operation_name_; + std::string trace_id_; + std::string parent_segment_id_; + std::string name_; + using Annotation = std::pair; + absl::flat_hash_map> annotations_; + std::vector> metadata_; + Tracing::Span* parent_span_; + uint64_t id_; +}; + class Tracer { public: Tracer(absl::string_view segment_name, TimeSource& time_source) @@ -22,15 +154,19 @@ class Tracer { } /** - * Starts a tracing span for XRay + * Starts a tracing span for X-Ray */ - Tracing::SpanPtr startSpan() { return nullptr; } + Tracing::SpanPtr startSpan(const std::string& span_name, const std::string& operation_name, + Envoy::SystemTime start_time, + const absl::optional& xray_header); private: const std::string segment_name_; TimeSource& time_source_; }; +using TracerPtr = std::unique_ptr; + } // namespace XRay } // namespace Tracers } // namespace Extensions diff --git a/source/extensions/tracers/xray/xray_configuration.h b/source/extensions/tracers/xray/xray_configuration.h index c35c3f111fe0..852f70561498 100644 --- a/source/extensions/tracers/xray/xray_configuration.h +++ b/source/extensions/tracers/xray/xray_configuration.h @@ -1,24 +1,35 @@ #pragma once #include -#include "absl/strings/string_view.h" - namespace Envoy { namespace Extensions { namespace Tracers { namespace XRay { +/** + * X-Ray configuration Model. + */ struct XRayConfiguration { - XRayConfiguration(absl::string_view daemon_endpoint, absl::string_view segment_name, - absl::string_view sampling_rules) - : daemon_endpoint_(daemon_endpoint), segment_name_(segment_name), - sampling_rules_(sampling_rules) {} - const std::string daemon_endpoint_; const std::string segment_name_; const std::string sampling_rules_; }; +enum class SamplingDecision { + Sampled, + NotSampled, + Unknown, +}; + +/** + * X-Ray header Model. + */ +struct XRayHeader { + std::string trace_id_; + std::string parent_id_; + SamplingDecision sample_decision_; +}; + } // namespace XRay } // namespace Tracers } // namespace Extensions diff --git a/source/extensions/tracers/xray/xray_tracer_impl.cc b/source/extensions/tracers/xray/xray_tracer_impl.cc index 841eb1d0428d..141932a93da4 100644 --- a/source/extensions/tracers/xray/xray_tracer_impl.cc +++ b/source/extensions/tracers/xray/xray_tracer_impl.cc @@ -1,44 +1,110 @@ #include "extensions/tracers/xray/xray_tracer_impl.h" #include "common/common/macros.h" +#include "common/common/utility.h" + +#include "extensions/tracers/xray/localized_sampling.h" +#include "extensions/tracers/xray/tracer.h" +#include "extensions/tracers/xray/xray_configuration.h" namespace Envoy { namespace Extensions { namespace Tracers { namespace XRay { -static const char DefaultDaemonEndpoint[] = "127.0.0.1:2000"; -Driver::Driver(const XRayConfiguration& config, Server::Instance& server) : xray_config_(config) { +constexpr static char DefaultDaemonEndpoint[] = "127.0.0.1:2000"; +static XRayHeader parseXRayHeader(const Http::LowerCaseString& header) { + const auto& lowered_header = header.get(); + XRayHeader result; + for (const auto& token : StringUtil::splitToken(lowered_header, ";")) { + if (absl::StartsWith(token, "root=")) { + result.trace_id_ = std::string(StringUtil::cropLeft(token, "=")); + } else if (absl::StartsWith(token, "parent=")) { + result.parent_id_ = std::string(StringUtil::cropLeft(token, "=")); + } else if (absl::StartsWith(token, "sampled=")) { + const auto s = StringUtil::cropLeft(token, "="); + if (s == "1") { + result.sample_decision_ = SamplingDecision::Sampled; + } else if (s == "0") { + result.sample_decision_ = SamplingDecision::NotSampled; + } else { + result.sample_decision_ = SamplingDecision::Unknown; + } + } + } + return result; +} + +Driver::Driver(const XRayConfiguration& config, Server::Instance& server) + : xray_config_(config), tls_slot_ptr_(server.threadLocal().allocateSlot()) { const std::string daemon_endpoint = config.daemon_endpoint_.empty() ? DefaultDaemonEndpoint : config.daemon_endpoint_; ENVOY_LOG(debug, "send X-Ray generated segments to daemon address on {}", daemon_endpoint); - std::string span_name = - config.segment_name_.empty() ? server.localInfo().clusterName() : config.segment_name_; + sampling_strategy_ = std::make_unique( + xray_config_.sampling_rules_, server.random(), server.timeSource()); - sampling_strategy_ = std::make_unique(server.random().random()); - tracer_.emplace(span_name, server.timeSource()); + tls_slot_ptr_->set([this, + &server](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + std::string span_name = xray_config_.segment_name_.empty() ? server.localInfo().clusterName() + : xray_config_.segment_name_; + TracerPtr tracer = std::make_unique(span_name, server.timeSource()); + return std::make_shared(std::move(tracer), *this); + }); } Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, Http::HeaderMap& request_headers, - const std::string& operation_name, SystemTime start_time, + const std::string& operation_name, Envoy::SystemTime start_time, const Tracing::Decision tracing_decision) { + // First thing is to determine whether this request will be sampled or not. + // if there's a X-Ray header and it has a sampling decision already determined (i.e. Sample=1) + // then we can skip, otherwise, we ask the sampling strategy whether this request should be + // sampled or not. + // + // The second step is create a Span. + // If we have a XRay TraceID in the headers, then we create a SpanContext to pass that trace-id + // around if no TraceID (which means no x-ray header) then this is a brand new span. UNREFERENCED_PARAMETER(config); - UNREFERENCED_PARAMETER(operation_name); UNREFERENCED_PARAMETER(start_time); + // TODO(marcomagdy) - how do we factor this into the logic above UNREFERENCED_PARAMETER(tracing_decision); - const SamplingRequest request{request_headers.Host()->value().getStringView(), - request_headers.Method()->value().getStringView(), - request_headers.Path()->value().getStringView()}; + const auto* header = request_headers.get(Http::LowerCaseString(XRayTraceHeader)); + absl::optional should_trace; + XRayHeader xray_header; + if (header) { + Http::LowerCaseString lowered_header_value{std::string(header->value().getStringView())}; + xray_header = parseXRayHeader(lowered_header_value); + // if the sample_decision in the x-ray header is unknown then we try to make a decision based + // on the sampling strategy + if (xray_header.sample_decision_ == SamplingDecision::Sampled) { + should_trace = true; + } else if (xray_header.sample_decision_ == SamplingDecision::NotSampled) { + should_trace = false; + } else { + ENVOY_LOG( + trace, + "Unable to determine from the X-Ray trace header whether request is sampled or not"); + } + } + + if (!should_trace.has_value()) { + const SamplingRequest request{std::string{request_headers.Host()->value().getStringView()}, + std::string{request_headers.Method()->value().getStringView()}, + std::string{request_headers.Path()->value().getStringView()}}; - if (!sampling_strategy_->sampleRequest(request)) { - return nullptr; + should_trace = sampling_strategy_->shouldTrace(request); } - return tracer_->startSpan(); + if (should_trace.value()) { + auto tracer = tls_slot_ptr_->getTyped().tracer_.get(); + return tracer->startSpan(xray_config_.segment_name_, operation_name, start_time, + header ? absl::optional(xray_header) : absl::nullopt); + } + return nullptr; } + } // namespace XRay } // namespace Tracers } // namespace Extensions diff --git a/source/extensions/tracers/xray/xray_tracer_impl.h b/source/extensions/tracers/xray/xray_tracer_impl.h index c1c4dd84d785..76bb0afaaa93 100644 --- a/source/extensions/tracers/xray/xray_tracer_impl.h +++ b/source/extensions/tracers/xray/xray_tracer_impl.h @@ -1,6 +1,7 @@ #pragma once #include "envoy/server/instance.h" +#include "envoy/thread_local/thread_local.h" #include "envoy/tracing/http_tracer.h" #include "common/tracing/http_tracer_impl.h" @@ -13,18 +14,24 @@ namespace Extensions { namespace Tracers { namespace XRay { -class Driver : public Tracing::Driver, Logger::Loggable { +class Driver : public Tracing::Driver, public Logger::Loggable { public: Driver(const XRay::XRayConfiguration& config, Server::Instance& server); Tracing::SpanPtr startSpan(const Tracing::Config& config, Http::HeaderMap& request_headers, - const std::string& operation_name, SystemTime start_time, + const std::string& operation_name, Envoy::SystemTime start_time, const Tracing::Decision tracing_decision) override; private: + struct TlsTracer : ThreadLocal::ThreadLocalObject { + TlsTracer(TracerPtr tracer, Driver& driver) : tracer_(std::move(tracer)), driver_(driver) {} + TracerPtr tracer_; + Driver& driver_; + }; + XRayConfiguration xray_config_; SamplingStrategyPtr sampling_strategy_; - absl::optional tracer_; // optional<> for delayed construction + ThreadLocal::SlotPtr tls_slot_ptr_; }; } // namespace XRay diff --git a/test/extensions/tracers/xray/BUILD b/test/extensions/tracers/xray/BUILD index 30f262320bc5..2ee6b8f70e57 100644 --- a/test/extensions/tracers/xray/BUILD +++ b/test/extensions/tracers/xray/BUILD @@ -15,11 +15,22 @@ envoy_package() envoy_extension_cc_test( name = "xray_test", srcs = [ + "localized_sampling_test.cc", "util_test.cc", + "xray_tracer_impl_test.cc", ], extension_name = "envoy.tracers.xray", deps = [ "//source/extensions/tracers/xray:xray_lib", + "//test/mocks:common_lib", + "//test/mocks/http:http_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/mocks/server:server_mocks", + "//test/mocks/stats:stats_mocks", + "//test/mocks/thread_local:thread_local_mocks", + "//test/mocks/tracing:tracing_mocks", + "//test/test_common:simulated_time_system_lib", ], ) diff --git a/test/extensions/tracers/xray/localized_sampling_test.cc b/test/extensions/tracers/xray/localized_sampling_test.cc new file mode 100644 index 000000000000..1d291ab60306 --- /dev/null +++ b/test/extensions/tracers/xray/localized_sampling_test.cc @@ -0,0 +1,612 @@ +#include "extensions/tracers/xray/localized_sampling.h" + +#include "test/mocks/runtime/mocks.h" +#include "test/test_common/simulated_time_system.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace XRay { + +namespace { + +class LocalizedSamplingStrategyTest : public ::testing::Test { +protected: + Event::SimulatedTimeSystem time_system_; +}; + +TEST_F(LocalizedSamplingStrategyTest, EmptyRules) { + NiceMock random_generator; + LocalizedSamplingStrategy strategy{"", random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, BadJson) { + NiceMock random_generator; + LocalizedSamplingStrategy strategy{"{{}", random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, ValidCustomRules) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0, + "rate": 0.05 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_FALSE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, InvalidRate) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0, + "rate": 1.5 + } + ], + "default": { + "fixed_target": 1, + "rate": 0 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, InvalidFixedTarget) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 4.2, + "rate": 0.1 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, DefaultRuleMissingRate) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0, + "rate": 0.05 + } + ], + "default": { + "fixed_target": 1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, DefaultRuleMissingFixedTarget) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0, + "rate": 0.05 + } + ], + "default": { + "rate": 0.5 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, WrongVersion) { + NiceMock random_generator; + constexpr auto wrong_version = R"EOF( +{ + "version": 1, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0, + "rate": 0.05 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.5 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{wrong_version, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, MissingVersion) { + NiceMock random_generator; + constexpr auto missing_version = R"EOF( +{ + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0, + "rate": 0.05 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.5 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{missing_version, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, MissingDefaultRules) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0, + "rate": 0.05 + } + ] +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, CustomRuleHostIsNotString) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": null, + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0, + "rate": 0.05 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, CustomRuleHttpMethodIsNotString) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": 42, + "url_path": "/api/move/*", + "fixed_target": 0, + "rate": 0.05 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, CustomRuleUrlPathIsNotString) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": { "another": "object" }, + "fixed_target": 0, + "rate": 0.05 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, CustomRuleMissingFixedTarget) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "rate": 0.05 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, CustomRuleMissingRate) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, CustomRuleArrayElementWithWrongType) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0 + }, + "should be an array, not string" + ], + "default": { + "fixed_target": 1, + "rate": 0.1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, CustomRuleNegativeFixedRate) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": -1, + "rate": 0.05 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, CustomRuleNegativeRate) { + NiceMock random_generator; + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/api/move/*", + "fixed_target": 0, + "rate": 0.05 + } + ], + "default": { + "fixed_target": 1, + "rate": -0.1 + } +} + )EOF"; + LocalizedSamplingStrategy strategy{rules_json, random_generator, time_system_}; + ASSERT_TRUE(strategy.usingDefaultManifest()); +} + +TEST_F(LocalizedSamplingStrategyTest, TraceOnlyFromReservoir) { + NiceMock rng; + EXPECT_CALL(rng, random()).WillRepeatedly(Return(90)); + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "*", + "fixed_target": 1, + "rate": 0.5 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.5 + } +} + )EOF"; + + LocalizedSamplingStrategy strategy{rules_json, rng, time_system_}; + ASSERT_FALSE(strategy.usingDefaultManifest()); + + SamplingRequest req; + ASSERT_TRUE(strategy.shouldTrace(req)); // first one should be traced + int i = 10; + while (i-- > 0) { + ASSERT_FALSE(strategy.shouldTrace(req)); + } +} + +TEST_F(LocalizedSamplingStrategyTest, TraceFromReservoirAndByRate) { + NiceMock rng; + EXPECT_CALL(rng, random()).WillRepeatedly(Return(1)); + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "*", + "fixed_target": 1, + "rate": 0.1 + } + ], + "default": { + "fixed_target": 1, + "rate": 0.5 + } +} + )EOF"; + + LocalizedSamplingStrategy strategy{rules_json, rng, time_system_}; + ASSERT_FALSE(strategy.usingDefaultManifest()); + + SamplingRequest req; + int i = 10; + while (i-- > 0) { + ASSERT_TRUE(strategy.shouldTrace(req)); + } +} + +TEST_F(LocalizedSamplingStrategyTest, NoMatchingHost) { + NiceMock rng; + // this following value doesn't affect the test + EXPECT_CALL(rng, random()).WillRepeatedly(Return(50 /*50 percent*/)); + // the following rules say: + // "Sample 1 request/sec then 90% of the requests there after. Requests must have example.com as + // its host" + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "Player moves.", + "host": "example.com", + "http_method": "*", + "url_path": "*", + "fixed_target": 1, + "rate": 0.9 + } + ], + "default": { + "fixed_target": 0, + "rate": 0 + } +} + )EOF"; + + LocalizedSamplingStrategy strategy{rules_json, rng, time_system_}; + ASSERT_FALSE(strategy.usingDefaultManifest()); + + SamplingRequest req; + req.host_ = "amazon.com"; // host does not match, so default rules apply. + int i = 10; + while (i-- > 0) { + ASSERT_FALSE(strategy.shouldTrace(req)); + } +} + +TEST_F(LocalizedSamplingStrategyTest, NoMatchingHttpMethod) { + NiceMock rng; + // this following value doesn't affect the test + EXPECT_CALL(rng, random()).WillRepeatedly(Return(50 /*50 percent*/)); + // the following rules say: + // "Sample 1 request/sec then 90% of the requests there after. Requests must have example.com as + // its host" + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "Player moves.", + "host": "*", + "http_method": "POST", + "url_path": "*", + "fixed_target": 1, + "rate": 0.9 + } + ], + "default": { + "fixed_target": 0, + "rate": 0 + } +} + )EOF"; + + LocalizedSamplingStrategy strategy{rules_json, rng, time_system_}; + ASSERT_FALSE(strategy.usingDefaultManifest()); + + SamplingRequest req; + req.http_method_ = "GET"; // method does not match, so default rules apply. + int i = 10; + while (i-- > 0) { + ASSERT_FALSE(strategy.shouldTrace(req)); + } +} + +TEST_F(LocalizedSamplingStrategyTest, NoMatchingPath) { + NiceMock rng; + // this following value doesn't affect the test + EXPECT_CALL(rng, random()).WillRepeatedly(Return(50 /*50 percent*/)); + // the following rules say: + // "Sample 1 request/sec then 90% of the requests there after. Requests must have example.com as + // its host" + constexpr auto rules_json = R"EOF( +{ + "version": 2, + "rules": [ + { + "description": "X-Ray rule", + "host": "*", + "http_method": "*", + "url_path": "/available/*", + "fixed_target": 1, + "rate": 0.9 + } + ], + "default": { + "fixed_target": 0, + "rate": 0 + } +} + )EOF"; + + LocalizedSamplingStrategy strategy{rules_json, rng, time_system_}; + ASSERT_FALSE(strategy.usingDefaultManifest()); + + SamplingRequest req; + req.http_url_ = "/"; // method does not match, so default rules apply. + int i = 10; + while (i-- > 0) { + ASSERT_FALSE(strategy.shouldTrace(req)); + } +} + +} // namespace +} // namespace XRay +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/xray/xray_tracer_impl_test.cc b/test/extensions/tracers/xray/xray_tracer_impl_test.cc new file mode 100644 index 000000000000..ad916775c0cb --- /dev/null +++ b/test/extensions/tracers/xray/xray_tracer_impl_test.cc @@ -0,0 +1,91 @@ +#include + +#include "extensions/tracers/xray/tracer.h" +#include "extensions/tracers/xray/xray_configuration.h" +#include "extensions/tracers/xray/xray_tracer_impl.h" + +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/server/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/mocks/tracing/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace XRay { + +namespace { + +class XRayDriverTest : public ::testing::Test { +public: + const std::string operation_name_ = "test_operation_name"; + NiceMock server_; + NiceMock tls_; + NiceMock tracing_config_; + Http::TestHeaderMapImpl request_headers_{ + {":authority", "api.amazon.com"}, {":path", "/"}, {":method", "GET"}}; +}; + +TEST_F(XRayDriverTest, XRayTraceHeaderNotSampled) { + request_headers_.addCopy(XRayTraceHeader, "Root=1-272793;Parent=5398ad8;Sampled=0"); + + XRayConfiguration config{"" /*daemon_endpoint*/, "test_segment_name", "" /*sampling_rules*/}; + Driver driver(config, server_); + + Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; + Envoy::SystemTime start_time; + auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + tracing_decision); + ASSERT_EQ(span, nullptr); +} + +TEST_F(XRayDriverTest, XRayTraceHeaderSampled) { + request_headers_.addCopy(XRayTraceHeader, "Root=1-272793;Parent=5398ad8;Sampled=1"); + + XRayConfiguration config{"" /*daemon_endpoint*/, "test_segment_name", "" /*sampling_rules*/}; + Driver driver(config, server_); + + Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; + Envoy::SystemTime start_time; + auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + tracing_decision); + ASSERT_NE(span, nullptr); +} + +TEST_F(XRayDriverTest, XRayTraceHeaderSamplingUnknown) { + request_headers_.addCopy(XRayTraceHeader, "Root=1-272793;Parent=5398ad8"); + + XRayConfiguration config{"" /*daemon_endpoint*/, "test_segment_name", "" /*sampling_rules*/}; + Driver driver(config, server_); + + Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; + Envoy::SystemTime start_time; + auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + tracing_decision); + ASSERT_NE(span, nullptr); +} + +TEST_F(XRayDriverTest, NoXRayTracerHeader) { + XRayConfiguration config{"" /*daemon_endpoint*/, "test_segment_name", "" /*sampling_rules*/}; + Driver driver(config, server_); + + Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; + Envoy::SystemTime start_time; + auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + tracing_decision); + // sampling should fall back to the default manifest since: + // a) there is no X-Ray header to determine the sampling decision + // b) there are sampling rules passed, so the default rules apply (1 req/sec and 5% after that + // within that second) + ASSERT_NE(span, nullptr); +} + +} // namespace +} // namespace XRay +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy From 648cfc3b6c1410d8b27aa33b6fc0392ab230d601 Mon Sep 17 00:00:00 2001 From: Ismo Puustinen Date: Tue, 19 Nov 2019 23:49:20 +0200 Subject: [PATCH 07/29] bazel: replace tclap mirror. (#9072) Description: Change (non-existent) https://github.com/eile/tclap/ to https://github.com/mirror/tclap/. I didn't validate the new mirror in any way, but the downloded release file sha256sum stays the same. In the future the releases should probably be downloaded from the real upstream. However the release 1.2.1 tarball from https://sourceforge.net/projects/tclap/files/ did not have the same sha256sum as the release file which Envoy uses. Risk Level: Medium Testing: N/A Docs Changes: N/A Release Notes: Fixes #9071 Signed-off-by: Ismo Puustinen --- bazel/repositories.bzl | 8 ++++---- bazel/repository_locations.bzl | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index b42eb34074ba..67c289b0f0e3 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -116,7 +116,7 @@ def envoy_dependencies(skip_targets = []): _com_github_circonus_labs_libcircllhist() _com_github_cyan4973_xxhash() _com_github_datadog_dd_opentracing_cpp() - _com_github_eile_tclap() + _com_github_mirror_tclap() _com_github_envoyproxy_sqlparser() _com_github_fmtlib_fmt() _com_github_gabime_spdlog() @@ -230,9 +230,9 @@ def _com_github_envoyproxy_sqlparser(): actual = "@com_github_envoyproxy_sqlparser//:sqlparser", ) -def _com_github_eile_tclap(): +def _com_github_mirror_tclap(): _repository_impl( - name = "com_github_eile_tclap", + name = "com_github_mirror_tclap", build_file = "@envoy//bazel/external:tclap.BUILD", patch_args = ["-p1"], # If and when we pick up tclap 1.4 or later release, @@ -243,7 +243,7 @@ def _com_github_eile_tclap(): ) native.bind( name = "tclap", - actual = "@com_github_eile_tclap//:tclap", + actual = "@com_github_mirror_tclap//:tclap", ) def _com_github_fmtlib_fmt(): diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 85d8923359ae..8a541c3590b6 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -78,10 +78,10 @@ REPOSITORY_LOCATIONS = dict( strip_prefix = "sql-parser-5f50c68bdf5f107692bb027d1c568f67597f4d7f", urls = ["https://github.com/envoyproxy/sql-parser/archive/5f50c68bdf5f107692bb027d1c568f67597f4d7f.tar.gz"], ), - com_github_eile_tclap = dict( + com_github_mirror_tclap = dict( sha256 = "f0ede0721dddbb5eba3a47385a6e8681b14f155e1129dd39d1a959411935098f", strip_prefix = "tclap-tclap-1-2-1-release-final", - urls = ["https://github.com/eile/tclap/archive/tclap-1-2-1-release-final.tar.gz"], + urls = ["https://github.com/mirror/tclap/archive/tclap-1-2-1-release-final.tar.gz"], ), com_github_fmtlib_fmt = dict( sha256 = "4c0741e10183f75d7d6f730b8708a99b329b2f942dad5a9da3385ab92bb4a15c", From 61201908c18b71633e4a8393c03978a83faac4d1 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Tue, 19 Nov 2019 16:22:44 -0800 Subject: [PATCH 08/29] rename deprecated call to logging.warn() to logging.warning(). (#9075) Risk Level: very low Testing: ./tools/check_format_test_helper.py --log=WARN Signed-off-by: Joshua Marantz --- tools/check_format_test_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check_format_test_helper.py b/tools/check_format_test_helper.py index d07dd67d212a..74bb3c7ccddb 100755 --- a/tools/check_format_test_helper.py +++ b/tools/check_format_test_helper.py @@ -281,4 +281,4 @@ def runChecks(): if errors != 0: logging.error("%d FAILURES" % errors) exit(1) - logging.warn("PASS") + logging.warning("PASS") From 27856582db4daa46bd1f10f4efacc76f42204d86 Mon Sep 17 00:00:00 2001 From: Dominik-K Date: Wed, 20 Nov 2019 16:58:55 +0100 Subject: [PATCH 09/29] docs/best_practices/edge: fix `use_proxy_proto` location (#9082) Description: use_proxy_proto is part of the listener.FilterChain (not HttpConnectionManager) Risk Level: None Testing: Manually tested with envoyproxy/envoy-alpine:v1.12.1 (Docker image) Docs Changes: see description Release Notes: docs: corrected use of use_proxy_proto in edge-proxy best-practice example Fix for #8001 Signed-off-by: Dominik --- docs/root/configuration/best_practices/edge.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/root/configuration/best_practices/edge.rst b/docs/root/configuration/best_practices/edge.rst index 94e1728eb172..b45639fe8072 100644 --- a/docs/root/configuration/best_practices/edge.rst +++ b/docs/root/configuration/best_practices/edge.rst @@ -72,14 +72,14 @@ The following is a YAML example of the above recommendation. tls_certificates: - certificate_chain: { filename: "example_com_cert.pem" } private_key: { filename: "example_com_key.pem" } + # Uncomment if Envoy is behind a load balancer that exposes client IP address using the PROXY protocol. + # use_proxy_proto: true filters: - name: envoy.http_connection_manager typed_config: "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager stat_prefix: ingress_http use_remote_address: true - # Uncomment if Envoy is behind a load balancer that exposes client IP address using the PROXY protocol. - # use_proxy_proto: true common_http_protocol_options: idle_timeout: 3600s # 1 hour http2_protocol_options: From b8e7d8dbfa98b3aae3d7ee884017770f9a6ce012 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 20 Nov 2019 11:00:00 -0500 Subject: [PATCH 10/29] ext_authz: Fix spelling mistake in type definition (#9066) Description: Fix a spelling mistake ("Con[s]tant") introduced in 2f5f947 Risk Level: Low Testing: git grep -i contant Docs Changes: N/A Release Notes: N/A Signed-off-by: Luke Shumaker --- source/extensions/filters/common/ext_authz/ext_authz.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/common/ext_authz/ext_authz.h b/source/extensions/filters/common/ext_authz/ext_authz.h index 3d0bc3ab9038..4a76a3f6da7a 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz.h +++ b/source/extensions/filters/common/ext_authz/ext_authz.h @@ -21,14 +21,14 @@ namespace ExtAuthz { /** * Constant values used for tracing metadata. */ -struct TracingContantValues { +struct TracingConstantValues { const std::string TraceStatus = "ext_authz_status"; const std::string TraceUnauthz = "ext_authz_unauthorized"; const std::string TraceOk = "ext_authz_ok"; const std::string HttpStatus = "ext_authz_http_status"; }; -using TracingConstants = ConstSingleton; +using TracingConstants = ConstSingleton; /** * Possible async results for a check call. From 0c5b3571c2d04f9de973012fd1b346aecb6ca5ba Mon Sep 17 00:00:00 2001 From: Kuat Date: Wed, 20 Nov 2019 18:24:28 -0800 Subject: [PATCH 11/29] grpc_stats: add protobuf serialization to filter object (#9034) serialize stream stats for telemetry Risk Level: low Testing: unit Signed-off-by: Kuat Yessenov --- .../config/filter/http/grpc_stats/v2alpha/config.proto | 9 +++++++++ .../http/http_filters/grpc_stats_filter.rst | 4 +++- .../filters/http/grpc_stats/grpc_stats_filter.h | 7 +++++++ test/extensions/filters/http/grpc_stats/config_test.cc | 6 ++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/api/envoy/config/filter/http/grpc_stats/v2alpha/config.proto b/api/envoy/config/filter/http/grpc_stats/v2alpha/config.proto index 20f856881827..609a80327664 100644 --- a/api/envoy/config/filter/http/grpc_stats/v2alpha/config.proto +++ b/api/envoy/config/filter/http/grpc_stats/v2alpha/config.proto @@ -18,3 +18,12 @@ message FilterConfig { // counts. bool emit_filter_state = 1; } + +// gRPC statistics filter state object in protobuf form. +message FilterObject { + // Count of request messages in the request stream. + uint64 request_message_count = 1; + + // Count of response messages in the response stream. + uint64 response_message_count = 2; +} diff --git a/docs/root/configuration/http/http_filters/grpc_stats_filter.rst b/docs/root/configuration/http/http_filters/grpc_stats_filter.rst index f09569b2682f..6c17f7e4c16c 100644 --- a/docs/root/configuration/http/http_filters/grpc_stats_filter.rst +++ b/docs/root/configuration/http/http_filters/grpc_stats_filter.rst @@ -4,8 +4,10 @@ gRPC Statistics =============== * gRPC :ref:`architecture overview ` -* :ref:`v2 API reference ` +* :ref:`v2 API reference ` * This filter should be configured with the name *envoy.filters.http.grpc_stats*. +* This filter can be enabled to emit a :ref:`filter state object + ` This is a filter which enables telemetry of gRPC calls. Additionally, the filter detects message boundaries in streaming gRPC calls and emits the message diff --git a/source/extensions/filters/http/grpc_stats/grpc_stats_filter.h b/source/extensions/filters/http/grpc_stats/grpc_stats_filter.h index f5058f19d45d..f2f6c6542986 100644 --- a/source/extensions/filters/http/grpc_stats/grpc_stats_filter.h +++ b/source/extensions/filters/http/grpc_stats/grpc_stats_filter.h @@ -17,6 +17,13 @@ namespace GrpcStats { struct GrpcStatsObject : public StreamInfo::FilterState::Object { uint64_t request_message_count = 0; uint64_t response_message_count = 0; + + ProtobufTypes::MessagePtr serializeAsProto() const override { + auto msg = std::make_unique(); + msg->set_request_message_count(request_message_count); + msg->set_response_message_count(response_message_count); + return msg; + } }; class GrpcStatsFilterConfig diff --git a/test/extensions/filters/http/grpc_stats/config_test.cc b/test/extensions/filters/http/grpc_stats/config_test.cc index 40c36053b501..133c0259b84c 100644 --- a/test/extensions/filters/http/grpc_stats/config_test.cc +++ b/test/extensions/filters/http/grpc_stats/config_test.cc @@ -173,6 +173,12 @@ TEST_F(GrpcStatsFilterConfigTest, MessageCounts) { .value()); EXPECT_EQ(2U, data.request_message_count); EXPECT_EQ(3U, data.response_message_count); + + auto filter_object = + *dynamic_cast( + data.serializeAsProto().get()); + EXPECT_EQ(2U, filter_object.request_message_count()); + EXPECT_EQ(3U, filter_object.response_message_count()); } } // namespace From a907cff53f6ffb33d9a87b5ef50934626caa1b9e Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 20 Nov 2019 21:42:24 -0800 Subject: [PATCH 12/29] api: Add mechanism for filters to define route action. (#8956) Add a mechanism for a filter to define the action for a route. Risk Level: Low Testing: N/A Docs Changes: Inline with proto change. Release Notes: N/A Fixes #8953. Signed-off-by: Mark D. Roth --- api/envoy/api/v2/route/route.proto | 12 +++++++++++- api/envoy/api/v3alpha/route/route.proto | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/api/envoy/api/v2/route/route.proto b/api/envoy/api/v2/route/route.proto index c28500e69efc..2810664e9040 100644 --- a/api/envoy/api/v2/route/route.proto +++ b/api/envoy/api/v2/route/route.proto @@ -152,6 +152,11 @@ message VirtualHost { google.protobuf.UInt32Value per_request_buffer_limit_bytes = 18; } +// A filter-defined action type. +message FilterAction { + google.protobuf.Any action = 1; +} + // A route is both a specification of how to match a request as well as an indication of what to do // next (e.g., redirect, forward, rewrite, etc.). // @@ -159,7 +164,7 @@ message VirtualHost { // // Envoy supports routing on HTTP method via :ref:`header matching // `. -// [#next-free-field: 17] +// [#next-free-field: 18] message Route { reserved 6; @@ -180,6 +185,11 @@ message Route { // Return an arbitrary HTTP response directly, without proxying. DirectResponseAction direct_response = 7; + + // [#not-implemented-hide:] + // If true, a filter will define the action (e.g., it could dynamically generate the + // RouteAction). + FilterAction filter_action = 17; } // The Metadata field can be used to provide additional information diff --git a/api/envoy/api/v3alpha/route/route.proto b/api/envoy/api/v3alpha/route/route.proto index 68c0911f9ecd..b0b409c71744 100644 --- a/api/envoy/api/v3alpha/route/route.proto +++ b/api/envoy/api/v3alpha/route/route.proto @@ -147,6 +147,11 @@ message VirtualHost { google.protobuf.UInt32Value per_request_buffer_limit_bytes = 18; } +// A filter-defined action type. +message FilterAction { + google.protobuf.Any action = 1; +} + // A route is both a specification of how to match a request as well as an indication of what to do // next (e.g., redirect, forward, rewrite, etc.). // @@ -154,7 +159,7 @@ message VirtualHost { // // Envoy supports routing on HTTP method via :ref:`header matching // `. -// [#next-free-field: 17] +// [#next-free-field: 18] message Route { reserved 6, 8; @@ -177,6 +182,11 @@ message Route { // Return an arbitrary HTTP response directly, without proxying. DirectResponseAction direct_response = 7; + + // [#not-implemented-hide:] + // If true, a filter will define the action (e.g., it could dynamically generate the + // RouteAction). + FilterAction filter_action = 17; } // The Metadata field can be used to provide additional information From b9f798dd1dec86d3ed067c449e3c3a49c57984b1 Mon Sep 17 00:00:00 2001 From: danzh Date: Thu, 21 Nov 2019 01:01:16 -0500 Subject: [PATCH 13/29] refact quic listener construction (#9088) Change how ActiveQuicListener is constructed to avoid seg fault caused by unspecified argument evaluation order in different compilers. Currently, it only works if compiler evaluates them from left to right. Apparently clang and certain version of GCC do so, but evaluation order is not guaranteed cross all compilers. If no, shared_ptr is moved before being dereferenced. Risk Level: low, code not is use Testing: existing test Signed-off-by: Dan Zhang --- .../quiche/active_quic_listener.cc | 19 ++++--------------- .../quiche/active_quic_listener.h | 5 ----- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/source/extensions/quic_listeners/quiche/active_quic_listener.cc b/source/extensions/quic_listeners/quiche/active_quic_listener.cc index 5edc0c0f51d4..8a3c5a3803d1 100644 --- a/source/extensions/quic_listeners/quiche/active_quic_listener.cc +++ b/source/extensions/quic_listeners/quiche/active_quic_listener.cc @@ -23,21 +23,10 @@ ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher, Network::SocketSharedPtr listen_socket, Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config) - : ActiveQuicListener(dispatcher, parent, *listen_socket, - std::make_unique(*listen_socket), - dispatcher.createUdpListener(std::move(listen_socket), *this), - listener_config, quic_config) {} - -ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher, - Network::ConnectionHandler& parent, - Network::Socket& listen_socket, - std::unique_ptr writer, - Network::UdpListenerPtr&& listener, - Network::ListenerConfig& listener_config, - const quic::QuicConfig& quic_config) : Server::ConnectionHandlerImpl::ActiveListenerImplBase(parent, listener_config), - udp_listener_(std::move(listener)), dispatcher_(dispatcher), - version_manager_(quic::CurrentSupportedVersions()), listen_socket_(listen_socket) { + dispatcher_(dispatcher), version_manager_(quic::CurrentSupportedVersions()), + listen_socket_(*listen_socket) { + udp_listener_ = dispatcher_.createUdpListener(std::move(listen_socket), *this); quic::QuicRandom* const random = quic::QuicRandom::GetInstance(); random->RandBytes(random_seed_, sizeof(random_seed_)); crypto_config_ = std::make_unique( @@ -51,7 +40,7 @@ ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher, crypto_config_.get(), quic_config, &version_manager_, std::move(connection_helper), std::move(alarm_factory), quic::kQuicDefaultConnectionIdLength, parent, config_, stats_, dispatcher, listen_socket_); - quic_dispatcher_->InitializeWithWriter(writer.release()); + quic_dispatcher_->InitializeWithWriter(new EnvoyQuicPacketWriter(listen_socket_)); } void ActiveQuicListener::onListenerShutdown() { diff --git a/source/extensions/quic_listeners/quiche/active_quic_listener.h b/source/extensions/quic_listeners/quiche/active_quic_listener.h index f502b6c35188..fc14a490e242 100644 --- a/source/extensions/quic_listeners/quiche/active_quic_listener.h +++ b/source/extensions/quic_listeners/quiche/active_quic_listener.h @@ -44,11 +44,6 @@ class ActiveQuicListener : public Network::UdpListenerCallbacks, private: friend class ActiveQuicListenerPeer; - ActiveQuicListener(Event::Dispatcher& dispatcher, Network::ConnectionHandler& parent, - Network::Socket& listen_socket, std::unique_ptr writer, - Network::UdpListenerPtr&& listener, Network::ListenerConfig& listener_config, - const quic::QuicConfig& quic_config); - Network::UdpListenerPtr udp_listener_; uint8_t random_seed_[16]; std::unique_ptr crypto_config_; From 9169b3d50f4ea7a0ac31a74e83e20af96386bc13 Mon Sep 17 00:00:00 2001 From: Marcello de Sales Date: Thu, 21 Nov 2019 08:07:23 -0800 Subject: [PATCH 14/29] Feature/examples/grpc bridge in 3 steps (#8932) Simplified gRPC bridge example into a fully-dockerized solution in 3 steps. Upgraded Go to 1.13 Using simpler Docker images Split into 4 containers for newcomers The step-by-step now is on the REDME.md Signed-off-by: Marcello de Sales --- examples/BUILD | 4 +- examples/grpc-bridge/.gitignore | 3 +- examples/grpc-bridge/Dockerfile-grpc | 7 - examples/grpc-bridge/Dockerfile-python | 10 - examples/grpc-bridge/README.md | 129 +++++++++ examples/grpc-bridge/client/Dockerfile | 18 ++ examples/grpc-bridge/client/client.py | 15 +- .../envoy-proxy.yaml} | 31 +- .../{bin/.gitkeep => client/kv/__init__.py} | 0 examples/grpc-bridge/client/kv_pb2.py | 266 ------------------ examples/grpc-bridge/client/requirements.txt | 3 +- .../grpc-bridge/docker-compose-protos.yaml | 21 ++ examples/grpc-bridge/docker-compose.yaml | 64 ++++- .../grpc-bridge/{service => }/protos/kv.proto | 2 +- examples/grpc-bridge/script/bootstrap.sh | 11 - examples/grpc-bridge/script/build.sh | 12 - examples/grpc-bridge/script/grpc_start.sh | 4 - examples/grpc-bridge/server/Dockerfile | 29 ++ .../envoy-proxy.yaml} | 33 +-- examples/grpc-bridge/server/go.mod | 11 + examples/grpc-bridge/server/go.sum | 47 ++++ examples/grpc-bridge/server/kv/go.mod | 0 .../{service/main.go => server/service.go} | 4 +- .../grpc-bridge/service/envoy-gen/kv_pb2.py | 266 ------------------ examples/grpc-bridge/service/gen/kv.pb.go | 237 ---------------- examples/grpc-bridge/service/gen/kv_pb2.py | 266 ------------------ .../grpc-bridge/service/gen/kv_pb2_grpc.py | 64 ----- examples/grpc-bridge/service/script/gen | 18 -- examples/grpc-bridge/service/script/start.sh | 3 - test/server/config_validation/server_test.cc | 2 +- 30 files changed, 362 insertions(+), 1218 deletions(-) delete mode 100644 examples/grpc-bridge/Dockerfile-grpc delete mode 100644 examples/grpc-bridge/Dockerfile-python create mode 100644 examples/grpc-bridge/client/Dockerfile rename examples/grpc-bridge/{config/s2s-python-envoy.yaml => client/envoy-proxy.yaml} (79%) rename examples/grpc-bridge/{bin/.gitkeep => client/kv/__init__.py} (100%) delete mode 100644 examples/grpc-bridge/client/kv_pb2.py create mode 100644 examples/grpc-bridge/docker-compose-protos.yaml rename examples/grpc-bridge/{service => }/protos/kv.proto (99%) delete mode 100755 examples/grpc-bridge/script/bootstrap.sh delete mode 100755 examples/grpc-bridge/script/build.sh delete mode 100755 examples/grpc-bridge/script/grpc_start.sh create mode 100644 examples/grpc-bridge/server/Dockerfile rename examples/grpc-bridge/{config/s2s-grpc-envoy.yaml => server/envoy-proxy.yaml} (62%) create mode 100644 examples/grpc-bridge/server/go.mod create mode 100644 examples/grpc-bridge/server/go.sum create mode 100644 examples/grpc-bridge/server/kv/go.mod rename examples/grpc-bridge/{service/main.go => server/service.go} (90%) delete mode 100644 examples/grpc-bridge/service/envoy-gen/kv_pb2.py delete mode 100644 examples/grpc-bridge/service/gen/kv.pb.go delete mode 100644 examples/grpc-bridge/service/gen/kv_pb2.py delete mode 100644 examples/grpc-bridge/service/gen/kv_pb2_grpc.py delete mode 100755 examples/grpc-bridge/service/script/gen delete mode 100644 examples/grpc-bridge/service/script/start.sh diff --git a/examples/BUILD b/examples/BUILD index cea6483751a4..299fece55584 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -16,8 +16,8 @@ filegroup( "cors/frontend/service-envoy.yaml", "front-proxy/front-envoy.yaml", "front-proxy/service-envoy.yaml", - "grpc-bridge/config/s2s-grpc-envoy.yaml", - "grpc-bridge/config/s2s-python-envoy.yaml", + "grpc-bridge/client/envoy-proxy.yaml", + "grpc-bridge/server/envoy-proxy.yaml", "jaeger-tracing/front-envoy-jaeger.yaml", "jaeger-tracing/service1-envoy-jaeger.yaml", "jaeger-tracing/service2-envoy-jaeger.yaml", diff --git a/examples/grpc-bridge/.gitignore b/examples/grpc-bridge/.gitignore index 0f1f1dfc6f0d..0e57261fc9a9 100644 --- a/examples/grpc-bridge/.gitignore +++ b/examples/grpc-bridge/.gitignore @@ -1,2 +1,3 @@ -bin/service .idea +server/kv/kv.pb.go +client/kv/kv_pb2.py diff --git a/examples/grpc-bridge/Dockerfile-grpc b/examples/grpc-bridge/Dockerfile-grpc deleted file mode 100644 index 679b0a728a01..000000000000 --- a/examples/grpc-bridge/Dockerfile-grpc +++ /dev/null @@ -1,7 +0,0 @@ -FROM envoyproxy/envoy-dev:latest - -RUN mkdir /var/log/envoy/ -COPY ./bin/service /usr/local/bin/srv -COPY ./script/grpc_start.sh /etc/grpc_start -CMD /etc/grpc_start - diff --git a/examples/grpc-bridge/Dockerfile-python b/examples/grpc-bridge/Dockerfile-python deleted file mode 100644 index e90c8a469a5c..000000000000 --- a/examples/grpc-bridge/Dockerfile-python +++ /dev/null @@ -1,10 +0,0 @@ -FROM envoyproxy/envoy-dev:latest - -RUN apt-get update -RUN apt-get -q install -y python-dev \ - python-pip -ADD ./client /client -RUN pip install -r /client/requirements.txt -RUN chmod a+x /client/client.py -RUN mkdir /var/log/envoy/ -CMD /usr/local/bin/envoy -c /etc/s2s-python-envoy.yaml diff --git a/examples/grpc-bridge/README.md b/examples/grpc-bridge/README.md index ba95cd45a444..5d28b5d48569 100644 --- a/examples/grpc-bridge/README.md +++ b/examples/grpc-bridge/README.md @@ -1,2 +1,131 @@ To learn about this sandbox and for instructions on how to run it please head over to the [envoy docs](https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/grpc_bridge) + +# gRPC HTTP/1.1 to HTTP/2 bridge + +This is an example of a key-value store where a client CLI, written in Python, updates a remote store, written in Go, using the stubs generated for both languages. + +Running clients that uses gRPC stubs and sends messages through a proxy +that upgrades the HTTP requests from http/1.1 to http/2. This is a more detailed +implementation of the Envoy documentation at https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/grpc_bridge + +* Client: talks in python and sends HTTP/1.1 requests (gRPC stubs) + * Client-Proxy: Envoy setup that acts as an egress and converts the HTTP/1.1 call to HTTP/2. +* Server: talks in golang and receives HTTP/2 requests (gRPC stubs) + * Server-Proxy: Envoy setup that acts as an ingress and receives the HTTP/2 calls + +`[client](http/1.1) -> [client-egress-proxy](http/2) -> [server-ingress-proxy](http/2) -> [server]` + +# Running in 3 Steps + +* Generate Stubs: both the `client` and `server` stubs in `python` and `go` respectively to be used by each server. +* Start Both Client and Server Servers and Proxies: ` +* Use the Client CLI to make calls to the kv server. + +## Generate Stubs + +* Uses the `protos` dir and generates the stubs for both `client` and `server` +* Inspect the file `docker-compose-protos.yaml` with the gRPC protoc commands to generate the stubs. + +```console +$ docker-compose -f docker-compose-protos.yaml up --remove-orphans +Starting grpc-bridge_stubs_python_1 ... done +Starting grpc-bridge_stubs_go_1 ... done +Attaching to grpc-bridge_stubs_go_1, grpc-bridge_stubs_python_1 +grpc-bridge_stubs_go_1 exited with code 0 +grpc-bridge_stubs_python_1 exited with code 0 +``` + +* The files created were the `kv` modules for both the client and server respective dir. + * Note that both stubs are their respective languages. + * For each language, use its ways to include the stubs as an external module. + +```console +$ ls -la client/kv/kv_pb2.py +-rw-r--r-- 1 mdesales CORP\Domain Users 9527 Nov 6 21:59 client/kv/kv_pb2.py + +$ ls -la server/kv/kv.pb.go +-rw-r--r-- 1 mdesales CORP\Domain Users 9994 Nov 6 21:59 server/kv/kv.pb.go +``` + +## Start Both Client and Server and Proxies + +* After the stubs are in place, start the containers described in `docker-compose.yaml`. + +```console +$ docker-compose up --build +``` + +* Inspect the files `client/envoy-proxy.yaml` and `server/envoy-proxy.yaml`, as they define configs for their respective container, comparing port numbers and other specific settings. + +Notice that you will be interacting with the client container, which hosts +the client python CLI. The port numbers for the proxies and the containers are displayed +by the `docker-compose ps`, so it's easier to compare with the `\*/envoy-proxy.yaml` config files for each +of the containers how they match. + +Note that the client container to use is `grpc-bridge_grpc-client_1` and binds to no port +as it will use the `python` CLI. + +```console +$ docker-compose ps + Name Command State Ports +------------------------------------------------------------------------------------------------------------------------------------ +grpc-bridge_grpc-client-proxy_1 /docker-entrypoint.sh /usr ... Up 10000/tcp, 0.0.0.0:9911->9911/tcp, 0.0.0.0:9991->9991/tcp +grpc-bridge_grpc-client_1 /bin/sh -c tail -f /dev/null Up +grpc-bridge_grpc-server-proxy_1 /docker-entrypoint.sh /usr ... Up 10000/tcp, 0.0.0.0:8811->8811/tcp, 0.0.0.0:8881->8881/tcp +grpc-bridge_grpc-server_1 /bin/sh -c /bin/server Up 0.0.0.0:8081->8081/tcp +``` + +## Use the Client CLI + +* Since the containers are running, you can use the client container to interact with the gRPC server through the proxies +* The client has the methods `set key value` and `get key` to use the in-memory key-value store. + +```console +$ docker-compose exec grpc-client /client/grpc-kv-client.py set foo bar +setf foo to bar +``` + +> NOTE: You could also run docker instead of docker-compose `docker exec -ti grpc-bridge_grpc-client_1 /client/grpc-kv-client.py set foo bar` + +* The server will display the gRPC call received by the server, and then the access logs from the proxy for the SET method. + * Note that the proxy is propagating the headers of the request + +```console +grpc-server_1 | 2019/11/07 16:33:58 set: foo = bar +grpc-server-proxy_1 | [2019-11-07T16:33:58.856Z] "POST /kv.KV/Set HTTP/1.1" 200 - 15 7 3 1 "172.24.0.3" "python-requests/2.22.0" "c11cf735-0647-4e67-965c-5b1e362a5532" "grpc" "172.24.0.2:8081" +grpc-client-proxy_1 | [2019-11-07T16:33:58.855Z] "POST /kv.KV/Set HTTP/1.1" 200 - 15 7 5 3 "172.24.0.3" "python-requests/2.22.0" "c11cf735-0647-4e67-965c-5b1e362a5532" "grpc" "172.24.0.5:8811" +``` + +* Getting the value is no different + +```console +$ docker-compose exec grpc-client /client/grpc-kv-client.py get foo +bar +``` + +> NOTE: You could also run docker instead of docker-compose `docker exec -ti grpc-bridge_grpc-client_1 /client/grpc-kv-client.py get foo` + +* The logs in the server will show the same for the GET method. + * Note that again the request ID is proxied through + +```console +grpc-server_1 | 2019/11/07 16:34:50 get: foo +grpc-server-proxy_1 | [2019-11-07T16:34:50.456Z] "POST /kv.KV/Get HTTP/1.1" 200 - 10 10 2 1 "172.24.0.3" "python-requests/2.22.0" "727d4dcd-a276-4bb2-b4cc-494ae7119c24" "grpc" "172.24.0.2:8081" +grpc-client-proxy_1 | [2019-11-07T16:34:50.455Z] "POST /kv.KV/Get HTTP/1.1" 200 - 10 10 3 2 "172.24.0.3" "python-requests/2.22.0" "727d4dcd-a276-4bb2-b4cc-494ae7119c24" "grpc" "172.24.0.5:8811" +``` + +# Troubleshooting + +* Errors building the `client` or `server` are related to the missing gRPC stubs. +* Make sure to produce the stubs before building + * The error below is when the server is missing the stubs in the kv dir. + +```console +$ go build -o server +go: finding github.com/envoyproxy/envoy/examples/grpc-bridge latest +go: finding github.com/envoyproxy/envoy/examples latest +go: finding github.com/envoyproxy/envoy/examples/grpc-bridge/server/kv latest +go: finding github.com/envoyproxy/envoy/examples/grpc-bridge/server latest +build github.com/envoyproxy/envoy: cannot load github.com/envoyproxy/envoy/examples/grpc-bridge/server/kv: no matching versions for query "latest" +``` diff --git a/examples/grpc-bridge/client/Dockerfile b/examples/grpc-bridge/client/Dockerfile new file mode 100644 index 000000000000..49425a773bc0 --- /dev/null +++ b/examples/grpc-bridge/client/Dockerfile @@ -0,0 +1,18 @@ +FROM grpc/python + +WORKDIR /client + +COPY requirements.txt /client/requirements.txt + +# Cache the dependencies +RUN pip install -r /client/requirements.txt + +# Copy the sources, including the stubs +COPY client.py /client/grpc-kv-client.py +COPY kv /client/kv + +RUN chmod a+x /client/grpc-kv-client.py + +# http://bigdatums.net/2017/11/07/how-to-keep-docker-containers-running/ +# Call docker exec /client/grpc.py set | get +CMD tail -f /dev/null diff --git a/examples/grpc-bridge/client/client.py b/examples/grpc-bridge/client/client.py index b1cba24bad2a..3c9f9d075509 100755 --- a/examples/grpc-bridge/client/client.py +++ b/examples/grpc-bridge/client/client.py @@ -1,17 +1,22 @@ #!/usr/bin/env python import requests, sys -import kv_pb2 as kv +import os + +# Stubs generated by protoc +from kv import kv_pb2 as kv + from struct import pack -HOST = "http://localhost:9001" +HOST = os.getenv('CLIENT_PROXY', "http://localhost:9001") HEADERS = {'content-type': 'application/grpc', 'Host': 'grpc'} USAGE = """ - -envoy-python-client usage: +grpc-client usage [{host}]: ./client.py set - sets the and ./client.py get - gets the value for - """ + + Set env var CLIENT_PROXY to change to a different host + """.format(host=HOST) class KVClient(): diff --git a/examples/grpc-bridge/config/s2s-python-envoy.yaml b/examples/grpc-bridge/client/envoy-proxy.yaml similarity index 79% rename from examples/grpc-bridge/config/s2s-python-envoy.yaml rename to examples/grpc-bridge/client/envoy-proxy.yaml index 5538654f031a..880065d4d80d 100644 --- a/examples/grpc-bridge/config/s2s-python-envoy.yaml +++ b/examples/grpc-bridge/client/envoy-proxy.yaml @@ -1,15 +1,9 @@ -admin: - access_log_path: "/var/log/envoy/admin_access.log" - address: - socket_address: - address: 0.0.0.0 - port_value: 9901 static_resources: listeners: - address: socket_address: - address: 127.0.0.1 - port_value: 9001 + address: 0.0.0.0 + port_value: 9911 filter_chains: - filters: - name: envoy.http_connection_manager @@ -22,38 +16,45 @@ static_resources: - name: envoy.file_access_log typed_config: "@type": type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog - path: "/var/log/envoy/admin_access.log" + path: "/dev/stdout" stat_prefix: egress_http use_remote_address: true route_config: name: local_route virtual_hosts: - - name: grpc + - name: backend domains: - grpc routes: - match: prefix: "/" route: - cluster: grpc + cluster: backend-proxy http_filters: - name: envoy.grpc_http1_bridge typed_config: {} - name: envoy.router typed_config: {} clusters: - - name: grpc + - name: backend-proxy type: logical_dns dns_lookup_family: V4_ONLY lb_policy: round_robin connect_timeout: 0.250s http_protocol_options: {} load_assignment: - cluster_name: grpc + cluster_name: backend-proxy endpoints: - lb_endpoints: - endpoint: address: socket_address: - address: grpc - port_value: 9211 + address: kv-backend-proxy + port_value: 8811 + +admin: + access_log_path: "/tmp/admin_access.log" + address: + socket_address: + address: 0.0.0.0 + port_value: 9991 diff --git a/examples/grpc-bridge/bin/.gitkeep b/examples/grpc-bridge/client/kv/__init__.py similarity index 100% rename from examples/grpc-bridge/bin/.gitkeep rename to examples/grpc-bridge/client/kv/__init__.py diff --git a/examples/grpc-bridge/client/kv_pb2.py b/examples/grpc-bridge/client/kv_pb2.py deleted file mode 100644 index 40e5f27bc606..000000000000 --- a/examples/grpc-bridge/client/kv_pb2.py +++ /dev/null @@ -1,266 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: kv.proto - -import sys -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - -DESCRIPTOR = _descriptor.FileDescriptor( - name='kv.proto', - package='kv', - syntax='proto3', - serialized_pb=_b( - '\n\x08kv.proto\x12\x02kv\"\x19\n\nGetRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\"\x1c\n\x0bGetResponse\x12\r\n\x05value\x18\x01 \x01(\t\"(\n\nSetRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x19\n\x0bSetResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x32T\n\x02KV\x12&\n\x03Get\x12\x0e.kv.GetRequest\x1a\x0f.kv.GetResponse\x12&\n\x03Set\x12\x0e.kv.SetRequest\x1a\x0f.kv.SetResponseb\x06proto3' - )) - -_GETREQUEST = _descriptor.Descriptor( - name='GetRequest', - full_name='kv.GetRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', - full_name='kv.GetRequest.key', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=16, - serialized_end=41, -) - -_GETRESPONSE = _descriptor.Descriptor( - name='GetResponse', - full_name='kv.GetResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='value', - full_name='kv.GetResponse.value', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=43, - serialized_end=71, -) - -_SETREQUEST = _descriptor.Descriptor( - name='SetRequest', - full_name='kv.SetRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', - full_name='kv.SetRequest.key', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', - full_name='kv.SetRequest.value', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=73, - serialized_end=113, -) - -_SETRESPONSE = _descriptor.Descriptor( - name='SetResponse', - full_name='kv.SetResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='ok', - full_name='kv.SetResponse.ok', - index=0, - number=1, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=115, - serialized_end=140, -) - -DESCRIPTOR.message_types_by_name['GetRequest'] = _GETREQUEST -DESCRIPTOR.message_types_by_name['GetResponse'] = _GETRESPONSE -DESCRIPTOR.message_types_by_name['SetRequest'] = _SETREQUEST -DESCRIPTOR.message_types_by_name['SetResponse'] = _SETRESPONSE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -GetRequest = _reflection.GeneratedProtocolMessageType( - 'GetRequest', - (_message.Message,), - dict( - DESCRIPTOR=_GETREQUEST, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.GetRequest) - )) -_sym_db.RegisterMessage(GetRequest) - -GetResponse = _reflection.GeneratedProtocolMessageType( - 'GetResponse', - (_message.Message,), - dict( - DESCRIPTOR=_GETRESPONSE, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.GetResponse) - )) -_sym_db.RegisterMessage(GetResponse) - -SetRequest = _reflection.GeneratedProtocolMessageType( - 'SetRequest', - (_message.Message,), - dict( - DESCRIPTOR=_SETREQUEST, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.SetRequest) - )) -_sym_db.RegisterMessage(SetRequest) - -SetResponse = _reflection.GeneratedProtocolMessageType( - 'SetResponse', - (_message.Message,), - dict( - DESCRIPTOR=_SETRESPONSE, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.SetResponse) - )) -_sym_db.RegisterMessage(SetResponse) - -_KV = _descriptor.ServiceDescriptor( - name='KV', - full_name='kv.KV', - file=DESCRIPTOR, - index=0, - options=None, - serialized_start=142, - serialized_end=226, - methods=[ - _descriptor.MethodDescriptor( - name='Get', - full_name='kv.KV.Get', - index=0, - containing_service=None, - input_type=_GETREQUEST, - output_type=_GETRESPONSE, - options=None, - ), - _descriptor.MethodDescriptor( - name='Set', - full_name='kv.KV.Set', - index=1, - containing_service=None, - input_type=_SETREQUEST, - output_type=_SETRESPONSE, - options=None, - ), - ]) -_sym_db.RegisterServiceDescriptor(_KV) - -DESCRIPTOR.services_by_name['KV'] = _KV - -# @@protoc_insertion_point(module_scope) diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index 8ef4a95ccde0..42f680cc4639 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -1,3 +1,4 @@ requests>=2.22.0 grpcio -protobuf==3.8.0 +grpcio-tools +protobuf==3.10.0 diff --git a/examples/grpc-bridge/docker-compose-protos.yaml b/examples/grpc-bridge/docker-compose-protos.yaml new file mode 100644 index 000000000000..6b791161231d --- /dev/null +++ b/examples/grpc-bridge/docker-compose-protos.yaml @@ -0,0 +1,21 @@ +version: "3" + +# This is the conversion from a script to a dockerized version of the script +# https://github.com/envoyproxy/envoy/blob/master/examples/grpc-bridge/service/script/gen +services: + + # $ docker run -ti -v $(pwd):/protos -v $(pwd)/stubs:/stubs grpc/go protoc --go_out=plugins=grpc:/stubs -I/protos /protos/kv.proto + stubs_go: + image: grpc/go + command: protoc --go_out=plugins=grpc:/stubs -I/protos /protos/kv.proto + volumes: + - ./protos:/protos + - ./server/kv:/stubs + + # $ docker run -ti -v $(pwd):/protos -v $(pwd)/stubs:/stubs grpc/python python -m grpc.tools.protoc --python_out=/stubs --grpc_python_out=/stubs -I/protos /protos/kv.proto + stubs_python: + image: grpc/python + command: python -m grpc.tools.protoc --python_out=/stubs --grpc_python_out=/stubs -I/protos /protos/kv.proto + volumes: + - ./protos:/protos + - ./client/kv:/stubs diff --git a/examples/grpc-bridge/docker-compose.yaml b/examples/grpc-bridge/docker-compose.yaml index 25d5b1fb3cd3..c09707a310e5 100644 --- a/examples/grpc-bridge/docker-compose.yaml +++ b/examples/grpc-bridge/docker-compose.yaml @@ -1,20 +1,64 @@ version: "3.7" + services: - python: + # Requires the build of the stubs first + grpc-server: + image: envoyproxy/example-kv-server build: - context: . - dockerfile: Dockerfile-python + context: server + expose: + - "8081" + ports: + - "8081:8081" + networks: + envoymesh: + aliases: + - kv-backend-service + + grpc-server-proxy: + image: envoyproxy/envoy:latest + command: /usr/local/bin/envoy -c /etc/server-envoy-proxy.yaml --service-cluster backend-proxy volumes: - - ./config/s2s-python-envoy.yaml:/etc/s2s-python-envoy.yaml + - ./server/envoy-proxy.yaml:/etc/server-envoy-proxy.yaml + networks: + envoymesh: + aliases: + - kv-backend-proxy expose: - - "9001" + - "8811" + - "8881" + ports: + - "8811:8811" + - "8881:8881" - grpc: + # Requires the build of the stubs first + grpc-client: + image: envoyproxy/example-kv-client build: - context: . - dockerfile: Dockerfile-grpc + context: client + environment: + - CLIENT_PROXY=http://kv-client-proxy:9911 + networks: + envoymesh: + aliases: + - grpc-client + + grpc-client-proxy: + image: envoyproxy/envoy:latest + command: /usr/local/bin/envoy -c /etc/client-envoy-proxy.yaml volumes: - - ./config/s2s-grpc-envoy.yaml:/etc/s2s-grpc-envoy.yaml + - ./client/envoy-proxy.yaml:/etc/client-envoy-proxy.yaml + networks: + envoymesh: + aliases: + - kv-client-proxy expose: - - "9211" + - "9911" + - "9991" + ports: + - "9911:9911" + - "9991:9991" + +networks: + envoymesh: {} diff --git a/examples/grpc-bridge/service/protos/kv.proto b/examples/grpc-bridge/protos/kv.proto similarity index 99% rename from examples/grpc-bridge/service/protos/kv.proto rename to examples/grpc-bridge/protos/kv.proto index 774f75683c89..584e89b4f271 100644 --- a/examples/grpc-bridge/service/protos/kv.proto +++ b/examples/grpc-bridge/protos/kv.proto @@ -22,4 +22,4 @@ message SetResponse { service KV { rpc Get(GetRequest) returns (GetResponse); rpc Set(SetRequest) returns (SetResponse); -} \ No newline at end of file +} diff --git a/examples/grpc-bridge/script/bootstrap.sh b/examples/grpc-bridge/script/bootstrap.sh deleted file mode 100755 index d8ba5a4f242a..000000000000 --- a/examples/grpc-bridge/script/bootstrap.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -e - -cd $(dirname $0)/.. - -echo "fetching dependencies..." -go get golang.org/x/net/context -go get golang.org/x/sys/unix -go get google.golang.org/grpc -echo "done" diff --git a/examples/grpc-bridge/script/build.sh b/examples/grpc-bridge/script/build.sh deleted file mode 100755 index d3ed6dda0575..000000000000 --- a/examples/grpc-bridge/script/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -cd $(dirname $0)/.. - -export GOOS=linux -export GOARCH=amd64 -export CGO_ENABLED=0 - -go build -o ./bin/service ./service - diff --git a/examples/grpc-bridge/script/grpc_start.sh b/examples/grpc-bridge/script/grpc_start.sh deleted file mode 100755 index 76147577c6cd..000000000000 --- a/examples/grpc-bridge/script/grpc_start.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -srv & -/usr/local/bin/envoy -c /etc/s2s-grpc-envoy.yaml diff --git a/examples/grpc-bridge/server/Dockerfile b/examples/grpc-bridge/server/Dockerfile new file mode 100644 index 000000000000..2c13ec006a12 --- /dev/null +++ b/examples/grpc-bridge/server/Dockerfile @@ -0,0 +1,29 @@ +FROM golang:1.13.0-stretch as builder + +WORKDIR /build + +# Resolve and build Go dependencies as Docker cache +COPY go.mod /build/go.mod +COPY go.sum /build/go.sum +COPY kv/go.mod /build/kv/go.mod + +ENV GO111MODULE=on +RUN go mod download + +COPY service.go /build/main.go +COPY kv/ /build/kv + +# Build for linux +ENV GOOS=linux +ENV GOARCH=amd64 +ENV CGO_ENABLED=0 +RUN go build -o server + +# Build the main container (Linux Runtime) +FROM alpine:latest +WORKDIR /root/ + +# Copy the linux amd64 binary +COPY --from=builder /build/server /bin/ + +ENTRYPOINT /bin/server diff --git a/examples/grpc-bridge/config/s2s-grpc-envoy.yaml b/examples/grpc-bridge/server/envoy-proxy.yaml similarity index 62% rename from examples/grpc-bridge/config/s2s-grpc-envoy.yaml rename to examples/grpc-bridge/server/envoy-proxy.yaml index 0f5e794739b8..fea386492836 100644 --- a/examples/grpc-bridge/config/s2s-grpc-envoy.yaml +++ b/examples/grpc-bridge/server/envoy-proxy.yaml @@ -3,7 +3,7 @@ static_resources: - address: socket_address: address: 0.0.0.0 - port_value: 9211 + port_value: 8811 filter_chains: - filters: - name: envoy.http_connection_manager @@ -11,10 +11,15 @@ static_resources: "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager codec_type: auto stat_prefix: ingress_http + access_log: + - name: envoy.file_access_log + typed_config: + "@type": type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog + path: "/dev/stdout" route_config: name: local_route virtual_hosts: - - name: local_service + - name: backend domains: - "*" routes: @@ -22,28 +27,24 @@ static_resources: prefix: "/" grpc: {} route: - cluster: local_service_grpc + cluster: backend_grpc_service http_filters: - name: envoy.router typed_config: {} clusters: - - name: local_service_grpc + - name: backend_grpc_service connect_timeout: 0.250s - type: static + type: strict_dns lb_policy: round_robin http2_protocol_options: {} - load_assignment: - cluster_name: local_service_grpc - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 8081 + hosts: + - socket_address: + address: kv-backend-service + port_value: 8081 + admin: - access_log_path: "/var/log/envoy/admin_access.log" + access_log_path: "/tmp/admin_access.log" address: socket_address: address: 0.0.0.0 - port_value: 9901 + port_value: 8881 diff --git a/examples/grpc-bridge/server/go.mod b/examples/grpc-bridge/server/go.mod new file mode 100644 index 000000000000..cb92ba66562d --- /dev/null +++ b/examples/grpc-bridge/server/go.mod @@ -0,0 +1,11 @@ +module github.com/envoyproxy/envoy + +go 1.13 + +require ( + github.com/envoyproxy/envoy/examples/grpc-bridge/server/kv v0.0.0-00010101000000-000000000000 + golang.org/x/net v0.0.0-20191105084925-a882066a44e0 + google.golang.org/grpc v1.25.0 +) + +replace github.com/envoyproxy/envoy/examples/grpc-bridge/server/kv => ./kv diff --git a/examples/grpc-bridge/server/go.sum b/examples/grpc-bridge/server/go.sum new file mode 100644 index 000000000000..634ba26b77ce --- /dev/null +++ b/examples/grpc-bridge/server/go.sum @@ -0,0 +1,47 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20191105084925-a882066a44e0 h1:QPlSTtPE2k6PZPasQUbzuK3p9JbS+vMXYVto8g/yrsg= +golang.org/x/net v0.0.0-20191105084925-a882066a44e0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.0 h1:ItERT+UbGdX+s4u+nQNlVM/Q7cbmf7icKfvzbWqVtq0= +google.golang.org/grpc v1.25.0/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/examples/grpc-bridge/server/kv/go.mod b/examples/grpc-bridge/server/kv/go.mod new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/examples/grpc-bridge/service/main.go b/examples/grpc-bridge/server/service.go similarity index 90% rename from examples/grpc-bridge/service/main.go rename to examples/grpc-bridge/server/service.go index 61037113bf21..7c4e5b8cbef5 100644 --- a/examples/grpc-bridge/service/main.go +++ b/examples/grpc-bridge/server/service.go @@ -8,7 +8,7 @@ import ( "sync" - "github.com/envoyproxy/envoy/examples/grpc-bridge/service/gen" + "github.com/envoyproxy/envoy/examples/grpc-bridge/server/kv" "golang.org/x/net/context" "google.golang.org/grpc" ) @@ -35,7 +35,7 @@ func (k *KV) Set(ctx context.Context, in *kv.SetRequest) (*kv.SetResponse, error k.store[in.Key] = in.Value - return &kv.SetResponse{true}, nil + return &kv.SetResponse{Ok: true}, nil } func NewKVStore() (kv *KV) { diff --git a/examples/grpc-bridge/service/envoy-gen/kv_pb2.py b/examples/grpc-bridge/service/envoy-gen/kv_pb2.py deleted file mode 100644 index 40e5f27bc606..000000000000 --- a/examples/grpc-bridge/service/envoy-gen/kv_pb2.py +++ /dev/null @@ -1,266 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: kv.proto - -import sys -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - -DESCRIPTOR = _descriptor.FileDescriptor( - name='kv.proto', - package='kv', - syntax='proto3', - serialized_pb=_b( - '\n\x08kv.proto\x12\x02kv\"\x19\n\nGetRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\"\x1c\n\x0bGetResponse\x12\r\n\x05value\x18\x01 \x01(\t\"(\n\nSetRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x19\n\x0bSetResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x32T\n\x02KV\x12&\n\x03Get\x12\x0e.kv.GetRequest\x1a\x0f.kv.GetResponse\x12&\n\x03Set\x12\x0e.kv.SetRequest\x1a\x0f.kv.SetResponseb\x06proto3' - )) - -_GETREQUEST = _descriptor.Descriptor( - name='GetRequest', - full_name='kv.GetRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', - full_name='kv.GetRequest.key', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=16, - serialized_end=41, -) - -_GETRESPONSE = _descriptor.Descriptor( - name='GetResponse', - full_name='kv.GetResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='value', - full_name='kv.GetResponse.value', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=43, - serialized_end=71, -) - -_SETREQUEST = _descriptor.Descriptor( - name='SetRequest', - full_name='kv.SetRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', - full_name='kv.SetRequest.key', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', - full_name='kv.SetRequest.value', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=73, - serialized_end=113, -) - -_SETRESPONSE = _descriptor.Descriptor( - name='SetResponse', - full_name='kv.SetResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='ok', - full_name='kv.SetResponse.ok', - index=0, - number=1, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=115, - serialized_end=140, -) - -DESCRIPTOR.message_types_by_name['GetRequest'] = _GETREQUEST -DESCRIPTOR.message_types_by_name['GetResponse'] = _GETRESPONSE -DESCRIPTOR.message_types_by_name['SetRequest'] = _SETREQUEST -DESCRIPTOR.message_types_by_name['SetResponse'] = _SETRESPONSE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -GetRequest = _reflection.GeneratedProtocolMessageType( - 'GetRequest', - (_message.Message,), - dict( - DESCRIPTOR=_GETREQUEST, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.GetRequest) - )) -_sym_db.RegisterMessage(GetRequest) - -GetResponse = _reflection.GeneratedProtocolMessageType( - 'GetResponse', - (_message.Message,), - dict( - DESCRIPTOR=_GETRESPONSE, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.GetResponse) - )) -_sym_db.RegisterMessage(GetResponse) - -SetRequest = _reflection.GeneratedProtocolMessageType( - 'SetRequest', - (_message.Message,), - dict( - DESCRIPTOR=_SETREQUEST, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.SetRequest) - )) -_sym_db.RegisterMessage(SetRequest) - -SetResponse = _reflection.GeneratedProtocolMessageType( - 'SetResponse', - (_message.Message,), - dict( - DESCRIPTOR=_SETRESPONSE, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.SetResponse) - )) -_sym_db.RegisterMessage(SetResponse) - -_KV = _descriptor.ServiceDescriptor( - name='KV', - full_name='kv.KV', - file=DESCRIPTOR, - index=0, - options=None, - serialized_start=142, - serialized_end=226, - methods=[ - _descriptor.MethodDescriptor( - name='Get', - full_name='kv.KV.Get', - index=0, - containing_service=None, - input_type=_GETREQUEST, - output_type=_GETRESPONSE, - options=None, - ), - _descriptor.MethodDescriptor( - name='Set', - full_name='kv.KV.Set', - index=1, - containing_service=None, - input_type=_SETREQUEST, - output_type=_SETRESPONSE, - options=None, - ), - ]) -_sym_db.RegisterServiceDescriptor(_KV) - -DESCRIPTOR.services_by_name['KV'] = _KV - -# @@protoc_insertion_point(module_scope) diff --git a/examples/grpc-bridge/service/gen/kv.pb.go b/examples/grpc-bridge/service/gen/kv.pb.go deleted file mode 100644 index 5ff0074cbd68..000000000000 --- a/examples/grpc-bridge/service/gen/kv.pb.go +++ /dev/null @@ -1,237 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: kv.proto - -/* -Package kv is a generated protocol buffer package. - -It is generated from these files: - kv.proto - -It has these top-level messages: - GetRequest - GetResponse - SetRequest - SetResponse -*/ -package kv - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" - -import ( - context "golang.org/x/net/context" - grpc "google.golang.org/grpc" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package - -type GetRequest struct { - Key string `protobuf:"bytes,1,opt,name=key" json:"key,omitempty"` -} - -func (m *GetRequest) Reset() { *m = GetRequest{} } -func (m *GetRequest) String() string { return proto.CompactTextString(m) } -func (*GetRequest) ProtoMessage() {} -func (*GetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } - -func (m *GetRequest) GetKey() string { - if m != nil { - return m.Key - } - return "" -} - -type GetResponse struct { - Value string `protobuf:"bytes,1,opt,name=value" json:"value,omitempty"` -} - -func (m *GetResponse) Reset() { *m = GetResponse{} } -func (m *GetResponse) String() string { return proto.CompactTextString(m) } -func (*GetResponse) ProtoMessage() {} -func (*GetResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } - -func (m *GetResponse) GetValue() string { - if m != nil { - return m.Value - } - return "" -} - -type SetRequest struct { - Key string `protobuf:"bytes,1,opt,name=key" json:"key,omitempty"` - Value string `protobuf:"bytes,2,opt,name=value" json:"value,omitempty"` -} - -func (m *SetRequest) Reset() { *m = SetRequest{} } -func (m *SetRequest) String() string { return proto.CompactTextString(m) } -func (*SetRequest) ProtoMessage() {} -func (*SetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } - -func (m *SetRequest) GetKey() string { - if m != nil { - return m.Key - } - return "" -} - -func (m *SetRequest) GetValue() string { - if m != nil { - return m.Value - } - return "" -} - -type SetResponse struct { - Ok bool `protobuf:"varint,1,opt,name=ok" json:"ok,omitempty"` -} - -func (m *SetResponse) Reset() { *m = SetResponse{} } -func (m *SetResponse) String() string { return proto.CompactTextString(m) } -func (*SetResponse) ProtoMessage() {} -func (*SetResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } - -func (m *SetResponse) GetOk() bool { - if m != nil { - return m.Ok - } - return false -} - -func init() { - proto.RegisterType((*GetRequest)(nil), "kv.GetRequest") - proto.RegisterType((*GetResponse)(nil), "kv.GetResponse") - proto.RegisterType((*SetRequest)(nil), "kv.SetRequest") - proto.RegisterType((*SetResponse)(nil), "kv.SetResponse") -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// Client API for KV service - -type KVClient interface { - Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error) - Set(ctx context.Context, in *SetRequest, opts ...grpc.CallOption) (*SetResponse, error) -} - -type kVClient struct { - cc *grpc.ClientConn -} - -func NewKVClient(cc *grpc.ClientConn) KVClient { - return &kVClient{cc} -} - -func (c *kVClient) Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error) { - out := new(GetResponse) - err := grpc.Invoke(ctx, "/kv.KV/Get", in, out, c.cc, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *kVClient) Set(ctx context.Context, in *SetRequest, opts ...grpc.CallOption) (*SetResponse, error) { - out := new(SetResponse) - err := grpc.Invoke(ctx, "/kv.KV/Set", in, out, c.cc, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// Server API for KV service - -type KVServer interface { - Get(context.Context, *GetRequest) (*GetResponse, error) - Set(context.Context, *SetRequest) (*SetResponse, error) -} - -func RegisterKVServer(s *grpc.Server, srv KVServer) { - s.RegisterService(&_KV_serviceDesc, srv) -} - -func _KV_Get_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(KVServer).Get(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/kv.KV/Get", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(KVServer).Get(ctx, req.(*GetRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _KV_Set_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(SetRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(KVServer).Set(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/kv.KV/Set", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(KVServer).Set(ctx, req.(*SetRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _KV_serviceDesc = grpc.ServiceDesc{ - ServiceName: "kv.KV", - HandlerType: (*KVServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Get", - Handler: _KV_Get_Handler, - }, - { - MethodName: "Set", - Handler: _KV_Set_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "kv.proto", -} - -func init() { proto.RegisterFile("kv.proto", fileDescriptor0) } - -var fileDescriptor0 = []byte{ - // 167 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xc8, 0x2e, 0xd3, 0x2b, - 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xca, 0x2e, 0x53, 0x92, 0xe3, 0xe2, 0x72, 0x4f, 0x2d, 0x09, - 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x12, 0xe0, 0x62, 0xce, 0x4e, 0xad, 0x94, 0x60, 0x54, - 0x60, 0xd4, 0xe0, 0x0c, 0x02, 0x31, 0x95, 0x94, 0xb9, 0xb8, 0xc1, 0xf2, 0xc5, 0x05, 0xf9, 0x79, - 0xc5, 0xa9, 0x42, 0x22, 0x5c, 0xac, 0x65, 0x89, 0x39, 0xa5, 0xa9, 0x50, 0x25, 0x10, 0x8e, 0x92, - 0x09, 0x17, 0x57, 0x30, 0x1e, 0x43, 0x10, 0xba, 0x98, 0x90, 0x75, 0xc9, 0x72, 0x71, 0x07, 0x23, - 0x19, 0xcd, 0xc7, 0xc5, 0x94, 0x9f, 0x0d, 0xd6, 0xc5, 0x11, 0xc4, 0x94, 0x9f, 0x6d, 0x14, 0xc2, - 0xc5, 0xe4, 0x1d, 0x26, 0xa4, 0xc6, 0xc5, 0xec, 0x9e, 0x5a, 0x22, 0xc4, 0xa7, 0x97, 0x5d, 0xa6, - 0x87, 0x70, 0xa8, 0x14, 0x3f, 0x9c, 0x0f, 0xd5, 0xad, 0xc6, 0xc5, 0x1c, 0x0c, 0x53, 0x17, 0x8c, - 0xa6, 0x0e, 0xc9, 0x96, 0x24, 0x36, 0xb0, 0xd7, 0x8d, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0x4a, - 0x17, 0xca, 0xf0, 0x06, 0x01, 0x00, 0x00, -} diff --git a/examples/grpc-bridge/service/gen/kv_pb2.py b/examples/grpc-bridge/service/gen/kv_pb2.py deleted file mode 100644 index 40e5f27bc606..000000000000 --- a/examples/grpc-bridge/service/gen/kv_pb2.py +++ /dev/null @@ -1,266 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: kv.proto - -import sys -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - -DESCRIPTOR = _descriptor.FileDescriptor( - name='kv.proto', - package='kv', - syntax='proto3', - serialized_pb=_b( - '\n\x08kv.proto\x12\x02kv\"\x19\n\nGetRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\"\x1c\n\x0bGetResponse\x12\r\n\x05value\x18\x01 \x01(\t\"(\n\nSetRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x19\n\x0bSetResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x32T\n\x02KV\x12&\n\x03Get\x12\x0e.kv.GetRequest\x1a\x0f.kv.GetResponse\x12&\n\x03Set\x12\x0e.kv.SetRequest\x1a\x0f.kv.SetResponseb\x06proto3' - )) - -_GETREQUEST = _descriptor.Descriptor( - name='GetRequest', - full_name='kv.GetRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', - full_name='kv.GetRequest.key', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=16, - serialized_end=41, -) - -_GETRESPONSE = _descriptor.Descriptor( - name='GetResponse', - full_name='kv.GetResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='value', - full_name='kv.GetResponse.value', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=43, - serialized_end=71, -) - -_SETREQUEST = _descriptor.Descriptor( - name='SetRequest', - full_name='kv.SetRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', - full_name='kv.SetRequest.key', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', - full_name='kv.SetRequest.value', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=73, - serialized_end=113, -) - -_SETRESPONSE = _descriptor.Descriptor( - name='SetResponse', - full_name='kv.SetResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='ok', - full_name='kv.SetResponse.ok', - index=0, - number=1, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[], - serialized_start=115, - serialized_end=140, -) - -DESCRIPTOR.message_types_by_name['GetRequest'] = _GETREQUEST -DESCRIPTOR.message_types_by_name['GetResponse'] = _GETRESPONSE -DESCRIPTOR.message_types_by_name['SetRequest'] = _SETREQUEST -DESCRIPTOR.message_types_by_name['SetResponse'] = _SETRESPONSE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -GetRequest = _reflection.GeneratedProtocolMessageType( - 'GetRequest', - (_message.Message,), - dict( - DESCRIPTOR=_GETREQUEST, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.GetRequest) - )) -_sym_db.RegisterMessage(GetRequest) - -GetResponse = _reflection.GeneratedProtocolMessageType( - 'GetResponse', - (_message.Message,), - dict( - DESCRIPTOR=_GETRESPONSE, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.GetResponse) - )) -_sym_db.RegisterMessage(GetResponse) - -SetRequest = _reflection.GeneratedProtocolMessageType( - 'SetRequest', - (_message.Message,), - dict( - DESCRIPTOR=_SETREQUEST, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.SetRequest) - )) -_sym_db.RegisterMessage(SetRequest) - -SetResponse = _reflection.GeneratedProtocolMessageType( - 'SetResponse', - (_message.Message,), - dict( - DESCRIPTOR=_SETRESPONSE, - __module__='kv_pb2' - # @@protoc_insertion_point(class_scope:kv.SetResponse) - )) -_sym_db.RegisterMessage(SetResponse) - -_KV = _descriptor.ServiceDescriptor( - name='KV', - full_name='kv.KV', - file=DESCRIPTOR, - index=0, - options=None, - serialized_start=142, - serialized_end=226, - methods=[ - _descriptor.MethodDescriptor( - name='Get', - full_name='kv.KV.Get', - index=0, - containing_service=None, - input_type=_GETREQUEST, - output_type=_GETRESPONSE, - options=None, - ), - _descriptor.MethodDescriptor( - name='Set', - full_name='kv.KV.Set', - index=1, - containing_service=None, - input_type=_SETREQUEST, - output_type=_SETRESPONSE, - options=None, - ), - ]) -_sym_db.RegisterServiceDescriptor(_KV) - -DESCRIPTOR.services_by_name['KV'] = _KV - -# @@protoc_insertion_point(module_scope) diff --git a/examples/grpc-bridge/service/gen/kv_pb2_grpc.py b/examples/grpc-bridge/service/gen/kv_pb2_grpc.py deleted file mode 100644 index cecee7c882f6..000000000000 --- a/examples/grpc-bridge/service/gen/kv_pb2_grpc.py +++ /dev/null @@ -1,64 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -import grpc - -import kv_pb2 as kv__pb2 - - -class KVStub(object): - # missing associated documentation comment in .proto file - pass - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Get = channel.unary_unary( - '/kv.KV/Get', - request_serializer=kv__pb2.GetRequest.SerializeToString, - response_deserializer=kv__pb2.GetResponse.FromString, - ) - self.Set = channel.unary_unary( - '/kv.KV/Set', - request_serializer=kv__pb2.SetRequest.SerializeToString, - response_deserializer=kv__pb2.SetResponse.FromString, - ) - - -class KVServicer(object): - # missing associated documentation comment in .proto file - pass - - def Get(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Set(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_KVServicer_to_server(servicer, server): - rpc_method_handlers = { - 'Get': - grpc.unary_unary_rpc_method_handler( - servicer.Get, - request_deserializer=kv__pb2.GetRequest.FromString, - response_serializer=kv__pb2.GetResponse.SerializeToString, - ), - 'Set': - grpc.unary_unary_rpc_method_handler( - servicer.Set, - request_deserializer=kv__pb2.SetRequest.FromString, - response_serializer=kv__pb2.SetResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler('kv.KV', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) diff --git a/examples/grpc-bridge/service/script/gen b/examples/grpc-bridge/service/script/gen deleted file mode 100755 index bed331f80c1d..000000000000 --- a/examples/grpc-bridge/service/script/gen +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -cd $(dirname $0)/.. - -rm -rf gen/* - -# generate the protobufs -protoc --go_out=plugins=grpc:./gen \ - -I./protos ./protos/kv.proto - -python -m grpc.tools.protoc \ - -I./protos \ - --python_out=./gen \ - --grpc_python_out=./gen ./protos/kv.proto - -cp ./gen/kv_pb2.py ../client/ diff --git a/examples/grpc-bridge/service/script/start.sh b/examples/grpc-bridge/service/script/start.sh deleted file mode 100644 index 1a57f89f1836..000000000000 --- a/examples/grpc-bridge/service/script/start.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -srv diff --git a/test/server/config_validation/server_test.cc b/test/server/config_validation/server_test.cc index ea1d201efce2..a25fcf321fb2 100644 --- a/test/server/config_validation/server_test.cc +++ b/test/server/config_validation/server_test.cc @@ -76,7 +76,7 @@ TEST_P(ValidationServerTest, NoopLifecycleNotifier) { INSTANTIATE_TEST_SUITE_P(ValidConfigs, ValidationServerTest, ::testing::Values("front-proxy_front-envoy.yaml", "google_com_proxy.v2.yaml", - "grpc-bridge_config_s2s-grpc-envoy.yaml", + "grpc-bridge_server_envoy-proxy.yaml", "front-proxy_service-envoy.yaml")); // Just make sure that all configs can be ingested without a crash. Processing of config files From 37885bb570b81d68c8412d31521eb650842e5969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20B=C3=A9ky?= Date: Thu, 21 Nov 2019 15:55:59 -0500 Subject: [PATCH 15/29] tls: roll QUICHE to 9711a9e74b43 and BoringSSL to 65e0aad1b721 (#8856) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Description: roll QUICHE and BoringSSL external dependencies to more recent versions and make necessary modifications. Risk Level: Low. Testing: No new tests. Both external dependencies are covered by extensive unit tests. No new functionality is added to Envoy, therefore existing integration tests should be sufficient. Docs Changes: n/a. Release Notes: n/a. Signed-off-by: Bence Béky --- bazel/external/quiche.BUILD | 113 ++++--- bazel/repository_locations.bzl | 14 +- .../quiche/envoy_quic_dispatcher.h | 14 +- .../quiche/envoy_quic_server_stream.cc | 4 +- .../quic_listeners/quiche/platform/BUILD | 3 - .../quiche/platform/flags_list.h | 287 ++++++++---------- .../quiche/platform/http2_ptr_util_impl.h | 18 -- .../quiche/platform/http2_string_impl.h | 15 - .../quiche/platform/quic_export_impl.h | 1 + .../quiche/platform/quic_mutex_impl.h | 11 + .../quiche/platform/quic_string_impl.h | 15 - .../quiche/platform/spdy_containers_impl.h | 1 - .../quiche/platform/spdy_string_impl.h | 15 - .../quiche/envoy_quic_server_session_test.cc | 3 +- .../quiche/envoy_quic_server_stream_test.cc | 15 +- .../quiche/platform/quic_platform_test.cc | 4 +- .../quiche/platform/quic_port_utils_impl.cc | 2 +- .../quiche/platform/quic_port_utils_impl.h | 2 +- 18 files changed, 235 insertions(+), 302 deletions(-) delete mode 100644 source/extensions/quic_listeners/quiche/platform/http2_ptr_util_impl.h delete mode 100644 source/extensions/quic_listeners/quiche/platform/http2_string_impl.h delete mode 100644 source/extensions/quic_listeners/quiche/platform/quic_string_impl.h delete mode 100644 source/extensions/quic_listeners/quiche/platform/spdy_string_impl.h diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 1b9e08f22145..9f5c7694fe8d 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -95,8 +95,6 @@ envoy_cc_library( "quiche/http2/platform/api/http2_logging.h", "quiche/http2/platform/api/http2_macros.h", "quiche/http2/platform/api/http2_optional.h", - "quiche/http2/platform/api/http2_ptr_util.h", - "quiche/http2/platform/api/http2_string.h", "quiche/http2/platform/api/http2_string_piece.h", "quiche/http2/platform/api/http2_string_utils.h", # TODO: uncomment the following files as implementations are added. @@ -719,7 +717,6 @@ envoy_cc_library( "quiche/spdy/platform/api/spdy_map_util.h", "quiche/spdy/platform/api/spdy_mem_slice.h", "quiche/spdy/platform/api/spdy_ptr_util.h", - "quiche/spdy/platform/api/spdy_string.h", "quiche/spdy/platform/api/spdy_string_piece.h", "quiche/spdy/platform/api/spdy_string_utils.h", ], @@ -1346,6 +1343,17 @@ envoy_cc_library( deps = [":quic_platform_export"], ) +envoy_cc_library( + name = "quic_core_coalesced_packet_lib", + srcs = ["quiche/quic/core/quic_coalesced_packet.cc"], + hdrs = ["quiche/quic/core/quic_coalesced_packet.h"], + copts = quiche_copts, + repository = "@envoy", + deps = [ + ":quic_core_packets_lib", + ], +) + envoy_cc_library( name = "quic_core_config_lib", srcs = ["quiche/quic/core/quic_config.cc"], @@ -1591,12 +1599,12 @@ envoy_cc_library( ":quic_core_crypto_crypto_handshake_lib", ":quic_core_crypto_encryption_lib", ":quic_core_framer_lib", + ":quic_core_mtu_discovery_lib", ":quic_core_one_block_arena_lib", ":quic_core_packet_creator_lib", ":quic_core_packet_generator_lib", ":quic_core_packet_writer_interface_lib", ":quic_core_packets_lib", - ":quic_core_pending_retransmission_lib", ":quic_core_proto_cached_network_parameters_proto_header", ":quic_core_sent_packet_manager_lib", ":quic_core_time_lib", @@ -1737,6 +1745,7 @@ envoy_cc_library( "quiche/quic/core/crypto/chacha_base_encrypter.cc", "quiche/quic/core/crypto/null_decrypter.cc", "quiche/quic/core/crypto/null_encrypter.cc", + "quiche/quic/core/crypto/quic_crypter.cc", "quiche/quic/core/crypto/quic_decrypter.cc", "quiche/quic/core/crypto/quic_encrypter.cc", ], @@ -1964,6 +1973,7 @@ envoy_cc_library( ":quic_core_error_codes_lib", ":quic_core_interval_lib", ":quic_core_types_lib", + ":quic_core_versions_lib", ":quic_platform_base", ":quic_platform_mem_slice_span", ], @@ -2001,6 +2011,7 @@ envoy_cc_library( ":quic_core_crypto_encryption_lib", ":quic_core_http_spdy_session_lib", ":quic_core_packets_lib", + ":quic_core_qpack_qpack_streams_lib", ":quic_core_server_id_lib", ":quic_core_session_lib", ":quic_core_types_lib", @@ -2040,6 +2051,7 @@ envoy_cc_library( ":quic_core_data_lib", ":quic_core_error_codes_lib", ":quic_core_http_http_frames_lib", + ":quic_core_http_spdy_utils_lib", ":quic_core_types_lib", ":quic_platform_base", ], @@ -2056,6 +2068,7 @@ envoy_cc_library( ":quic_core_data_lib", ":quic_core_error_codes_lib", ":quic_core_http_http_frames_lib", + ":quic_core_http_spdy_utils_lib", ":quic_platform_base", ], ) @@ -2118,7 +2131,7 @@ envoy_cc_library( ":quic_core_http_http_constants_lib", ":quic_core_http_http_decoder_lib", ":quic_core_http_http_encoder_lib", - ":quic_core_http_spdy_stream_body_buffer_lib", + ":quic_core_http_spdy_stream_body_manager_lib", ":quic_core_http_spdy_utils_lib", ":quic_core_packets_lib", ":quic_core_proto_cached_network_parameters_proto_header", @@ -2127,6 +2140,7 @@ envoy_cc_library( ":quic_core_qpack_qpack_decoder_stream_sender_lib", ":quic_core_qpack_qpack_encoder_lib", ":quic_core_qpack_qpack_encoder_stream_sender_lib", + ":quic_core_qpack_qpack_streams_lib", ":quic_core_qpack_qpack_utils_lib", ":quic_core_session_lib", ":quic_core_utils_lib", @@ -2140,9 +2154,9 @@ envoy_cc_library( ) envoy_cc_library( - name = "quic_core_http_spdy_stream_body_buffer_lib", - srcs = ["quiche/quic/core/http/quic_spdy_stream_body_buffer.cc"], - hdrs = ["quiche/quic/core/http/quic_spdy_stream_body_buffer.h"], + name = "quic_core_http_spdy_stream_body_manager_lib", + srcs = ["quiche/quic/core/http/quic_spdy_stream_body_manager.cc"], + hdrs = ["quiche/quic/core/http/quic_spdy_stream_body_manager.h"], copts = quiche_copts, repository = "@envoy", tags = ["nofips"], @@ -2162,6 +2176,7 @@ envoy_cc_library( tags = ["nofips"], deps = [ ":quic_core_http_header_list_lib", + ":quic_core_http_http_constants_lib", ":quic_core_packets_lib", ":quic_platform_base", ":spdy_core_framer_lib", @@ -2199,6 +2214,17 @@ envoy_cc_library( deps = [":quic_platform_base"], ) +envoy_cc_library( + name = "quic_core_mtu_discovery_lib", + srcs = ["quiche/quic/core/quic_mtu_discovery.cc"], + hdrs = ["quiche/quic/core/quic_mtu_discovery.h"], + copts = quiche_copts, + repository = "@envoy", + deps = [ + ":quic_core_constants_lib", + ], +) + envoy_cc_library( name = "quic_core_one_block_arena_lib", srcs = ["quiche/quic/core/quic_one_block_arena.h"], @@ -2220,12 +2246,12 @@ envoy_cc_library( repository = "@envoy", tags = ["nofips"], deps = [ + ":quic_core_coalesced_packet_lib", ":quic_core_constants_lib", ":quic_core_crypto_encryption_lib", ":quic_core_data_lib", ":quic_core_framer_lib", ":quic_core_packets_lib", - ":quic_core_pending_retransmission_lib", ":quic_core_types_lib", ":quic_core_utils_lib", ":quic_core_versions_lib", @@ -2243,7 +2269,6 @@ envoy_cc_library( deps = [ ":quic_core_crypto_random_lib", ":quic_core_packet_creator_lib", - ":quic_core_pending_retransmission_lib", ":quic_core_sent_packet_manager_lib", ":quic_core_types_lib", ":quic_core_utils_lib", @@ -2314,19 +2339,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "quic_core_pending_retransmission_lib", - hdrs = ["quiche/quic/core/quic_pending_retransmission.h"], - repository = "@envoy", - tags = ["nofips"], - deps = [ - ":quic_core_frames_frames_lib", - ":quic_core_transmission_info_lib", - ":quic_core_types_lib", - ":quic_platform_export", - ], -) - envoy_cc_library( name = "quic_core_process_packet_interface_lib", hdrs = ["quiche/quic/core/quic_process_packet_interface.h"], @@ -2391,6 +2403,7 @@ envoy_cc_library( ":quic_core_qpack_qpack_decoder_stream_receiver_lib", ":quic_core_qpack_qpack_encoder_stream_sender_lib", ":quic_core_qpack_qpack_header_table_lib", + ":quic_core_qpack_qpack_index_conversions_lib", ":quic_core_qpack_qpack_instruction_encoder_lib", ":quic_core_qpack_qpack_required_insert_count_lib", ":quic_core_qpack_value_splitting_header_list_lib", @@ -2455,6 +2468,7 @@ envoy_cc_library( ":quic_core_qpack_qpack_decoder_stream_sender_lib", ":quic_core_qpack_qpack_encoder_stream_receiver_lib", ":quic_core_qpack_qpack_header_table_lib", + ":quic_core_qpack_qpack_index_conversions_lib", ":quic_core_qpack_qpack_instruction_decoder_lib", ":quic_core_qpack_qpack_required_insert_count_lib", ":quic_core_types_lib", @@ -2492,6 +2506,7 @@ envoy_cc_library( ":quic_core_qpack_qpack_constants_lib", ":quic_core_qpack_qpack_instruction_encoder_lib", ":quic_core_qpack_qpack_stream_sender_delegate_lib", + ":quic_core_types_lib", ":quic_platform_base", ], ) @@ -2547,6 +2562,18 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "quic_core_qpack_qpack_index_conversions_lib", + srcs = ["quiche/quic/core/qpack/qpack_index_conversions.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_index_conversions.h"], + copts = quiche_copts, + repository = "@envoy", + deps = [ + ":quic_platform_base", + ":quic_platform_export", + ], +) + envoy_cc_library( name = "quic_core_qpack_qpack_static_table_lib", srcs = ["quiche/quic/core/qpack/qpack_static_table.cc"], @@ -2569,6 +2596,25 @@ envoy_cc_library( deps = [":quic_platform_base"], ) +envoy_cc_library( + name = "quic_core_qpack_qpack_streams_lib", + srcs = [ + "quiche/quic/core/qpack/qpack_receive_stream.cc", + "quiche/quic/core/qpack/qpack_send_stream.cc", + ], + hdrs = [ + "quiche/quic/core/qpack/qpack_receive_stream.h", + "quiche/quic/core/qpack/qpack_send_stream.h", + ], + copts = quiche_copts, + repository = "@envoy", + deps = [ + ":quic_core_qpack_qpack_stream_receiver_lib", + ":quic_core_qpack_qpack_stream_sender_delegate_lib", + ":quic_core_session_lib", + ], +) + envoy_cc_library( name = "quic_core_qpack_qpack_decoded_headers_accumulator_lib", srcs = ["quiche/quic/core/qpack/qpack_decoded_headers_accumulator.cc"], @@ -2598,26 +2644,6 @@ envoy_cc_library( ], ) -# envoy_cc_library( -# name = "quic_core_qpack_qpack_streams_lib", -# srcs = [ -# "quiche/quic/core/qpack/qpack_receive_stream.cc", -# "quiche/quic/core/qpack/qpack_send_stream.cc", -# ], -# hdrs = [ -# "quiche/quic/core/qpack/qpack_receive_stream.h", -# "quiche/quic/core/qpack/qpack_send_stream.h", -# ], -# copts = quiche_copts, -# repository = "@envoy", -# deps = [ -# ":quic_core_http_spdy_session_lib", -# ":quic_core_qpack_qpack_stream_sender_delegate_lib", -# ":quic_core_session_lib", -# ":quic_platform_base", -# ], -# ) - envoy_cc_library( name = "quic_core_qpack_qpack_stream_sender_delegate_lib", hdrs = ["quiche/quic/core/qpack/qpack_stream_sender_delegate.h"], @@ -2661,7 +2687,6 @@ envoy_cc_library( ":quic_core_connection_stats_lib", ":quic_core_crypto_encryption_lib", ":quic_core_packets_lib", - ":quic_core_pending_retransmission_lib", ":quic_core_proto_cached_network_parameters_proto_header", ":quic_core_sustained_bandwidth_recorder_lib", ":quic_core_transmission_info_lib", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 8a541c3590b6..e916d616dc5b 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -27,15 +27,15 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/a6b28555badcb18d6be924c8fc1bea49971656b8.tar.gz"], ), boringssl = dict( - sha256 = "891352824e0f7977bc0c291b8c65076e3ed23630334841b93f346f12d4484b06", - strip_prefix = "boringssl-5565939d4203234ddc742c02241ce4523e7b3beb", + sha256 = "3eea198c8e3f587ffc8ea6acf87d7575f571bbe6dd88ec90405e236303f3dc01", + strip_prefix = "boringssl-65e0aad1b721a5aa67f2a8041cf48f691139bb9f", # To update BoringSSL, which tracks Chromium releases: # 1. Open https://omahaproxy.appspot.com/ and note of linux/beta release. # 2. Open https://chromium.googlesource.com/chromium/src/+/refs/tags//DEPS and note . # 3. Find a commit in BoringSSL's "master-with-bazel" branch that merges . # - # chromium-78.0.3904.21 (BETA) - urls = ["https://github.com/google/boringssl/archive/5565939d4203234ddc742c02241ce4523e7b3beb.tar.gz"], + # chromium-79.0.3945.16 (BETA) + urls = ["https://github.com/google/boringssl/archive/65e0aad1b721a5aa67f2a8041cf48f691139bb9f.tar.gz"], ), boringssl_fips = dict( sha256 = "b12ad676ee533824f698741bd127f6fbc82c46344398a6d78d25e62c6c418c73", @@ -278,9 +278,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://storage.googleapis.com/envoyproxy-wee8/wee8-7.9.317.14.tar.gz"], ), com_googlesource_quiche = dict( - # Static snapshot of https://quiche.googlesource.com/quiche/+archive/4abb566fbbc63df8fe7c1ac30b21632b9eb18d0c.tar.gz - sha256 = "c60bca3cf7f58b91394a89da96080657ff0fbe4d5675be9b21e90da8f68bc06f", - urls = ["https://storage.googleapis.com/quiche-envoy-integration/4abb566fbbc63df8fe7c1ac30b21632b9eb18d0c.tar.gz"], + # Static snapshot of https://quiche.googlesource.com/quiche/+archive/9711a9e74b43c7390ef7bb66c75561ff796900bf.tar.gz + sha256 = "2346dada2c1af2d5703a0d7350aa82465cd564fd15de115a1377792e771c43b4", + urls = ["https://storage.googleapis.com/quiche-envoy-integration/9711a9e74b43c7390ef7bb66c75561ff796900bf.tar.gz"], ), com_google_cel_cpp = dict( sha256 = "6b056207f6a069ee6e28f31010262585cf6090e6c889cb98da29715cf544ac7d", diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h index 64a5d0cd60bd..156311ec4b56 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h @@ -27,13 +27,6 @@ class EnvoyQuicCryptoServerStreamHelper : public quic::QuicCryptoServerStream::H ~EnvoyQuicCryptoServerStreamHelper() override = default; // quic::QuicCryptoServerStream::Helper - quic::QuicConnectionId - GenerateConnectionIdForReject(quic::QuicTransportVersion /*version*/, - quic::QuicConnectionId /*connection_id*/) const override { - // TODO(danzh): create reject connection id based on given connection_id. - return quic::QuicUtils::CreateRandomConnectionId(); - } - bool CanAcceptClientHello(const quic::CryptoHandshakeMessage& /*message*/, const quic::QuicSocketAddress& /*client_address*/, const quic::QuicSocketAddress& /*peer_address*/, @@ -62,6 +55,13 @@ class EnvoyQuicDispatcher : public quic::QuicDispatcher { const std::string& error_details, quic::ConnectionCloseSource source) override; + quic::QuicConnectionId + GenerateNewServerConnectionId(quic::ParsedQuicVersion /*version*/, + quic::QuicConnectionId /*connection_id*/) const override { + // TODO(danzh): create reject connection id based on given connection_id. + return quic::QuicUtils::CreateRandomConnectionId(); + } + protected: quic::QuicSession* CreateQuicSession(quic::QuicConnectionId server_connection_id, const quic::QuicSocketAddress& peer_address, diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc b/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc index b4ac61223b17..a815c2fbc4e8 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc @@ -184,7 +184,7 @@ void EnvoyQuicServerStream::OnBodyAvailable() { return; } - if (!quic::VersionUsesQpack(transport_version()) && !FinishedReadingTrailers()) { + if (!quic::VersionUsesHttp3(transport_version()) && !FinishedReadingTrailers()) { // For Google QUIC implementation, trailers may arrived earlier and wait to // be consumed after reading all the body. Consume it here. // IETF QUIC shouldn't reach here because trailers are sent on same stream. @@ -199,7 +199,7 @@ void EnvoyQuicServerStream::OnTrailingHeadersComplete(bool fin, size_t frame_len const quic::QuicHeaderList& header_list) { quic::QuicSpdyServerStreamBase::OnTrailingHeadersComplete(fin, frame_len, header_list); if (session()->connection()->connected() && - (quic::VersionUsesQpack(transport_version()) || sequencer()->IsClosed()) && + (quic::VersionUsesHttp3(transport_version()) || sequencer()->IsClosed()) && !FinishedReadingTrailers()) { // Before QPack trailers can arrive before body. Only decode trailers after finishing decoding // body. diff --git a/source/extensions/quic_listeners/quiche/platform/BUILD b/source/extensions/quic_listeners/quiche/platform/BUILD index eccf3089c3c4..4681f968ccc9 100644 --- a/source/extensions/quic_listeners/quiche/platform/BUILD +++ b/source/extensions/quic_listeners/quiche/platform/BUILD @@ -72,8 +72,6 @@ envoy_cc_library( "http2_logging_impl.h", "http2_macros_impl.h", "http2_optional_impl.h", - "http2_ptr_util_impl.h", - "http2_string_impl.h", "http2_string_piece_impl.h", "http2_string_utils_impl.h", ], @@ -293,7 +291,6 @@ envoy_cc_library( "spdy_map_util_impl.h", "spdy_mem_slice_impl.h", "spdy_ptr_util_impl.h", - "spdy_string_impl.h", "spdy_string_piece_impl.h", "spdy_string_utils_impl.h", "spdy_test_helpers_impl.h", diff --git a/source/extensions/quic_listeners/quiche/platform/flags_list.h b/source/extensions/quic_listeners/quiche/platform/flags_list.h index 11b8f3117150..b5cc4ecfc0b0 100644 --- a/source/extensions/quic_listeners/quiche/platform/flags_list.h +++ b/source/extensions/quic_listeners/quiche/platform/flags_list.h @@ -18,33 +18,32 @@ QUICHE_FLAG(bool, quic_reloadable_flag_advertise_quic_for_https_for_debugips, fa QUICHE_FLAG(bool, quic_reloadable_flag_advertise_quic_for_https_for_external_users, false, "") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_active_streams_never_negative, false, - "If true, static streams should never be closed before QuicSession " - "destruction.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_add_upper_limit_of_buffered_control_frames, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_add_upper_limit_of_buffered_control_frames3, true, "If true, close connection if there are too many (> 1000) buffered " "control frames.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_aggressive_connection_aliveness, false, - "If true, QuicSession::ShouldKeepConnectionAlive() will not consider " - "locally closed streams whose highest byte offset is not received yet.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_allow_backend_set_stream_ttl, false, "If true, check backend response header for X-Response-Ttl. If it is " "provided, the stream TTL is set. A QUIC stream will be immediately " "canceled when tries to write data if this TTL expired.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_allow_client_enabled_bbr_v2, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_allow_client_enabled_bbr_v2, true, "If true, allow client to enable BBRv2 on server via connection " "option 'B2ON'.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_alpn_dispatch, false, "Support different QUIC sessions, as indicated by ALPN. Used for QBONE.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_avoid_empty_frame_after_empty_headers, true, - "If enabled, do not call OnStreamFrame() with empty frame after " - "receiving empty or too large headers with FIN.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr2_fix_inflight_bounds, false, + "If true, for QUIC BBRv2: 1) don't grow inflight_hi unless it's " + "fully used, and 2) cap inflight_lo in PROBE_CRUISE.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr2_log_bbr2info_in_tracegraf, false, + "If true, for QUIC BBRv2 flows, log BBRv2-specific information to " + "tracegraf.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_fix_pacing_rate, false, + "If true, re-calculate pacing rate when cwnd gets bootstrapped.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_flexible_app_limited, false, "When true and the BBR9 connection option is present, BBR only considers " @@ -69,12 +68,20 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_startup_rate_reduction, false, "When true, enables the BBS4 and BBS5 connection options, which reduce " "BBR's pacing rate in STARTUP as more losses occur as a fraction of CWND.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_change_default_lumpy_pacing_size_to_two, false, - "If true and --quic_lumpy_pacing_size is 1, QUIC will use a lumpy " - "size of two for pacing.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_close_all_encryptions_levels, false, + "If true, QUIC connection close packet will be sent at all available " + "encryption levels.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_clear_queued_packets_on_connection_close, false, - "Calls ClearQueuedPackets after sending a connection close packet") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_close_connection_on_wrong_offset, false, + "If true, connection will be closed if a stream receives stream " + "frame or RESET_STREAM frame with bad close offset.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_coalesce_stream_frames, false, + "If true, Adjacent stream frames will be combined into one stream " + "frame before the packet is serialized.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_combine_generator_and_creator, true, + "If true, combine QuicPacketGenerator and QuicPacketCreator.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_conservative_bursts, false, "If true, set burst token to 2 in cwnd bootstrapping experiment.") @@ -97,144 +104,101 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_connection_migration_for_udp "If true, GFE disables connection migration in connection option for " "proxied sessions.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_39, false, - "If true, disable QUIC version 39.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_44, true, - "If true, disable QUIC version 44.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_do_not_accept_stop_waiting, false, "In v44 and above, where STOP_WAITING is never sent, close the " "connection if it's received.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_donot_process_small_initial_packets, true, + "If true, server drops client initial packets in datagrams < 1200 bytes.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_donot_reset_ideal_next_packet_send_time, false, "If true, stop resetting ideal_next_packet_send_time_ in pacing sender.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_drop_invalid_small_initial_connection_id, true, - "When true, QuicDispatcher will drop packets that have an initial " - "destination connection ID that is too short, instead of responding " - "with a Version Negotiation packet to reject it.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_eighth_rtt_loss_detection, false, - "When true, the LOSS connection option allows for 1/8 RTT of " - "reording instead of the current 1/8th threshold which has been " - "found to be too large for fast loss recovery.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_ack_decimation, false, "Default enables QUIC ack decimation and adds a connection option to " "disable it.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_fifo_write_scheduler, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_fifo_write_scheduler, true, "If true and FIFO connection option is received, write_blocked_streams " "uses FIFO(stream with smallest ID has highest priority) write scheduler.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_lifo_write_scheduler, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_ietf_loss_detection, true, + "If true, enable IETF loss detection as described in " + "https://tools.ietf.org/html/draft-ietf-quic-recovery-22#section-6.1.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_lifo_write_scheduler, true, "If true and LIFO connection option is received, write_blocked_streams " "uses LIFO(stream with largest ID has highest priority) write scheduler.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_pcc3, false, "If true, enable experiment for testing PCC congestion-control.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_47, false, - "If true, enable QUIC version 47 which adds support for variable " - "length connection IDs.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_pto, true, "If true, enable probe timeout.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_48, false, - "If true, enable QUIC version 48.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_rr_write_scheduler, true, + "If true, enable HTTP/2 default scheduling(round robin).") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_49, false, + "If true, enable QUIC version 49.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_50, false, + "If true, enable QUIC version 50.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_99, false, "If true, enable version 99.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_enabled, false, "") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_adaptive_time_loss, false, - "Simplify QUICHE's adaptive time loss detection to measure the " - "necessary reordering window for every spurious retransmit.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_bbr_cwnd_in_bandwidth_resumption, true, "If true, adjust congestion window when doing bandwidth resumption in BBR.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_get_packet_header_size, false, - "Fixes quic::GetPacketHeaderSize and callsites when " - "QuicVersionHasLongHeaderLengths is false.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_packets_acked, true, - "If true, when detecting losses, use packets_acked of corresponding " - "packet number space.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_rto_retransmission2, false, - "If true, when RTO fires and there is no packet to be RTOed, let " - "connection send.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_handle_staticness_for_spdy_stream, false, - "If true, QuicSpdySession::GetSpdyDataStream() will close the " - "connection if the returned stream is static.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_ignore_tlpr_if_no_pending_stream_data, false, - "If true, ignore TLPR if there is no pending stream data") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_inline_getorcreatedynamicstream, false, - "If true, QuicSession::GetOrCreateDynamicStream() is deprecated, and " - "its contents are moved to GetOrCreateStream().") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_framer_doesnt_create_initial_encrypter, true, + "If true, QuicFramer does not create an encrypter/decrypter for the " + "ENCRYPTION_INITIAL level.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_listener_never_fake_epollout, false, "If true, QuicListener::OnSocketIsWritable will always return false, " "which means there will never be a fake EPOLLOUT event in the next " "epoll iteration.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_log_cert_name_for_empty_sct, true, - "If true, log leaf cert subject name into warning log.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_loss_removes_from_inflight, true, - "When true, remove packets from inflight where they're declared " - "lost, rather than in MarkForRetransmission. Also no longer marks " - "handshake packets as no longer inflight when they're retransmitted.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_monotonic_epoll_clock, false, "If true, QuicEpollClock::Now() will monotonically increase.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_mtu_discovery_v2, false, + "If true, enable QUIC MTU discovery version 2.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_negotiate_ack_delay_time, false, "If true, will negotiate the ACK delay time.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_cloud_domain_sni_lookup_on_missing_sni, false, - "Do not attempt to match an empty Server Name Indication (SNI) " - "against names extracted from Cloud customer certificates.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_neuter_handshake_packets_once, false, + "Call NeuterHandshakePackets() at most once per connection.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_decrease_in_final_offset, false, + "If true, a stream will be reset if it receives fin that has offset " + "less than its highest offset.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_dup_experiment_id_2, false, "If true, transport connection stats doesn't report duplicated " "experiments for same connection.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_stream_data_after_reset, false, - "If true, QuicStreamSequencer will not take in new data if the stream is reset.") + "If true, QuicStreamSequencer will not take in new data if the " + "stream is reset.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_v2_scaling_factor, false, "When true, don't use an extra scaling factor when reading packets " - "from QUICHE's RX_RING with TPACKET_V2.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_window_update_on_read_only_stream, false, - "If true, QuicConnection will be closed if a WindowUpdate frame is " - "received on a READ_UNIDIRECTIONAL stream.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_proxy_check_toss_on_insertion_failure, false, - "If true, enable the code that fixes a race condition for quic udp " - "proxying in L0. See b/70036019.") + "from QUIC's RX_RING with TPACKET_V2.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_proxy_read_packed_strings, true, - "If true, QuicProxyDispatcher will prefer to extract client_address " - "and server_vip from packed_client_address and packed_server_vip, " - "respectively.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_parse_prox_source_connection_id, true, + "When true, QuicFramer allows parsing failures of source connection " + "ID for the PROX version.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_proxy_supports_length_prefix, false, - "When true, QuicProxyUtils::GetConnectionId supports length prefixed " - "connection IDs.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_populate_nonretransmittable_frames, false, + "If true, populate nonretransmittable frames in SerializedPacket.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_proxy_write_packed_strings, false, "If true, QuicProxyDispatcher will write packed_client_address and " "packed_server_vip in TcpProxyHeaderProto.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_length_prefix_from_packet_info, false, - "When true, QuicDispatcher::MaybeDispatchPacket will use packet_info.use_length_prefix " - "instead of an incorrect local computation.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_record_frontend_service_vip_mapping, false, "If true, for L1 GFE, as requests come in, record frontend service to VIP " "mapping which is used to announce VIP in SHLO for proxied sessions. ") @@ -254,31 +218,34 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_send_timestamps, false, "When the STMP connection option is sent by the client, timestamps " "in the QUIC ACK frame are sent and processed.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_sent_packet_manager_cleanup, false, - "When true, remove obsolete functionality intended to test IETF QUIC " - "recovery.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_server_push, true, - "If true, enable server push feature on QUICHE.") + "If true, enable server push feature on QUIC.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_simplify_stop_waiting, true, "If true, do not send STOP_WAITING if no_stop_waiting_frame_.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_skip_packet_number_for_pto, true, + "If true, skip packet number before sending the last PTO retransmission.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_stop_reading_when_level_triggered, false, "When true, calling StopReading() on a level-triggered QUIC stream " "sequencer will cause the sequencer to discard future data.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_supports_tls_handshake, false, + "If true, QUIC supports both QUIC Crypto and TLS 1.3 for the " + "handshake protocol.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_testonly_default_false, false, "A testonly reloadable flag that will always default to false.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_testonly_default_true, true, "A testonly reloadable flag that will always default to true.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_tracegraf_populate_ack_packet_number, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_tracegraf_populate_ack_packet_number, true, "If true, populate packet_number of received ACK in tracegraf.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_track_ack_height_in_bandwidth_sampler, false, - "If true, QUIC will track max ack height in BandwidthSampler.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_treat_queued_packets_as_sent, false, + "If true, treat queued QUIC packets as sent.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_unified_iw_options, false, "When true, set the initial congestion control window from connection " @@ -288,24 +255,21 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_common_stream_check, false, "If true, use common code for checking whether a new stream ID may " "be allocated.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_connection_clock_for_last_ack_time, false, - "If true, QuicFasterStatsGatherer will use a GFEConnectionClock to " - "get the time when the last ack is received.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_header_stage_idle_list2, false, "If true, use header stage idle list for QUIC connections in GFE.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_http2_priority_write_scheduler, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_http2_priority_write_scheduler, true, "If true and H2PR connection option is received, write_blocked_streams_ " "uses HTTP2 (tree-style) priority write scheduler.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_ip_bandwidth_module, true, + "If true, use IpBandwidthModule for cwnd bootstrapping if it is " + "registered.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_leto_key_exchange, false, "If true, QUIC will attempt to use the Leto key exchange service and " "only fall back to local key exchange if that fails.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_parse_public_header, false, - "When true, QuicDispatcher will always use QuicFramer::ParsePublicHeader") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_pigeon_sockets, false, "Use USPS Direct Path for QUIC egress.") @@ -322,12 +286,6 @@ QUICHE_FLAG(bool, quic_reloadable_flag_send_quic_fallback_server_config_on_leto_ "to a failure to contact Leto by sending a REJ containing a fallback " "ServerConfig, allowing the client to continue the handshake.") -QUICHE_FLAG(bool, quic_reloadable_flag_simplify_spdy_quic_https_scheme_detection, false, - "If true, simplify the logic for detecting REQUEST_HAS_HTTPS_SCHEME in " - "NetSpdyRequester::SetRequestUrlAndHost and " - "NetQuicRequester::SetRequestUrlAndHost. Fixes a bug where internal " - "redirects for QUIC connections would be treated as having an http scheme.") - QUICHE_FLAG(bool, quic_restart_flag_do_not_create_raw_socket_selector_if_quic_enabled, false, "If true, do not create the RawSocketSelector in " "QuicListener::Initialize() if QUIC is disabled by flag.") @@ -340,23 +298,9 @@ QUICHE_FLAG(bool, quic_restart_flag_quic_allow_loas_multipacket_chlo, false, "If true, inspects QUIC CHLOs for kLOAS and early creates sessions " "to allow multi-packet CHLOs") -QUICHE_FLAG(bool, quic_restart_flag_quic_connection_id_use_siphash, false, - "When true, QuicConnectionId::Hash uses SipHash instead of XOR.") - -QUICHE_FLAG(bool, quic_restart_flag_quic_dispatcher_hands_chlo_extractor_one_version, false, - "When true, QuicDispatcher will pass the version from the packet to " - "the ChloExtractor instead of all supported versions.") - -QUICHE_FLAG(bool, quic_restart_flag_quic_enable_gso_for_udp_egress, false, - "If 1) flag is true, 2) UDP egress_method is used in GFE config, and " - "3) UDP GSO is supported by the kernel, GFE will use UDP GSO for " - "egress, except for UDP proxy.") - -QUICHE_FLAG(bool, quic_restart_flag_quic_enable_sendmmsg_for_udp_egress, false, - "If 1) flag is true, 2) UDP egress_method is used in GFE config, and " - "3) --gfe2_restart_flag_quic_enable_gso_for_udp_egress is false OR " - "gso is not supported by kernel, GFE will use sendmmsg for egress, " - "except for UDP proxy.") +QUICHE_FLAG(bool, quic_restart_flag_quic_no_cap_net_raw_for_usps_egress, false, + "If true, gfe2::RawSocket::CapabilityNeeded will return false if " + "QUIC egress method is USPS.") QUICHE_FLAG(bool, quic_restart_flag_quic_no_fallback_for_pigeon_socket, false, "If true, GFEs using USPS egress will not fallback to raw ip socket.") @@ -364,27 +308,15 @@ QUICHE_FLAG(bool, quic_restart_flag_quic_no_fallback_for_pigeon_socket, false, QUICHE_FLAG(bool, quic_restart_flag_quic_offload_pacing_to_usps2, false, "If true, QUIC offload pacing when using USPS as egress method.") -QUICHE_FLAG(bool, quic_restart_flag_quic_pigeon_use_memfd_packet_memory, false, - "If true, GFE QUIC will forcefully use memfd to create packet memory " - "for pigeon socket. Otherwise memfd is used if " - "--pigeon_sealable_files_enabled is true.") - QUICHE_FLAG(bool, quic_restart_flag_quic_rx_ring_use_tpacket_v3, false, "If true, use TPACKET_V3 for QuicRxRing instead of TPACKET_V2.") -QUICHE_FLAG(bool, quic_restart_flag_quic_server_handle_egress_epoll_err, false, - "If true, handle EPOLLERRs from QUIC server egress sockets.") - QUICHE_FLAG(bool, quic_restart_flag_quic_testonly_default_false, false, "A testonly restart flag that will always default to false.") QUICHE_FLAG(bool, quic_restart_flag_quic_testonly_default_true, true, "A testonly restart flag that will always default to true.") -QUICHE_FLAG(bool, quic_restart_flag_quic_use_allocated_connection_ids, true, - "When true, QuicConnectionId will allocate long connection IDs on " - "the heap instead of inline in the object.") - QUICHE_FLAG(bool, quic_restart_flag_quic_use_leto_for_quic_configs, false, "If true, use Leto to fetch QUIC server configs instead of using the " "seeds from Memento.") @@ -398,7 +330,7 @@ QUICHE_FLAG(bool, quic_allow_chlo_buffering, true, "future CHLO, and allow CHLO packets to be buffered until next " "iteration of the event loop.") -QUICHE_FLAG(bool, quic_disable_pacing_for_perf_tests, false, "If true, disable pacing in QUICHE") +QUICHE_FLAG(bool, quic_disable_pacing_for_perf_tests, false, "If true, disable pacing in QUIC") QUICHE_FLAG(bool, quic_enforce_single_packet_chlo, true, "If true, enforce that QUIC CHLOs fit in one packet") @@ -430,13 +362,11 @@ QUICHE_FLAG(int32_t, // allow-non-std-int quic_send_buffer_max_data_slice_size, 4 * 1024, "Max size of data slice in bytes for QUIC stream send buffer.") -QUICHE_FLAG(bool, quic_supports_tls_handshake, false, - "If true, QUIC supports both QUIC Crypto and TLS 1.3 for the " - "handshake protocol") - QUICHE_FLAG(int32_t, // allow-non-std-int - quic_lumpy_pacing_size, 1, - "Number of packets that the pacing sender allows in bursts during pacing.") + quic_lumpy_pacing_size, 2, + "Number of packets that the pacing sender allows in bursts during " + "pacing. This flag is ignored if a flow's estimated bandwidth is " + "lower than 1200 kbps.") QUICHE_FLAG(double, quic_lumpy_pacing_cwnd_fraction, 0.25f, "Congestion window fraction that the pacing sender allows in bursts " @@ -450,10 +380,6 @@ QUICHE_FLAG(double, quic_pace_time_into_future_srtt_fraction, 0.125f, // One-eighth smoothed RTT "Smoothed RTT fraction that a connection can pace packets into the future.") -QUICHE_FLAG(int32_t, // allow-non-std-int - quic_ietf_draft_version, 0, - "Mechanism to override version label and ALPN for IETF interop.") - QUICHE_FLAG(bool, quic_export_server_num_packets_per_write_histogram, false, "If true, export number of packets written per write operation histogram.") @@ -484,10 +410,47 @@ QUICHE_FLAG(int32_t, // allow-non-std-int quic_bbr2_default_probe_rtt_period_ms, 10000, "The default period for entering PROBE_RTT, in milliseconds.") -QUICHE_FLAG(double, quic_bbr2_default_loss_threshold, 0.02, +QUICHE_FLAG(double, quic_bbr2_default_loss_threshold, + 0.3, // Changed from 0.02 for YouTube and Search experiments. "The default loss threshold for QUIC BBRv2, should be a value " "between 0 and 1.") +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_bbr2_default_startup_full_loss_count, 8, + "The default minimum number of loss marking events to exit STARTUP.") + +QUICHE_FLAG(double, quic_bbr2_default_inflight_hi_headroom, + 0.01, // Changed from 0.15 for YouTube and Search experiments. + "The default fraction of unutilized headroom to try to leave in path " + "upon high loss.") + +QUICHE_FLAG(double, quic_ack_aggregation_bandwidth_threshold, 1.0, + "If the bandwidth during ack aggregation is smaller than (estimated " + "bandwidth * this flag), consider the current aggregation completed " + "and starts a new one.") + +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_anti_amplification_factor, 3, + "Anti-amplification factor. Before address validation, server will " + "send no more than factor times bytes received.") + +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_max_buffered_crypto_bytes, + 16 * 1024, // 16 KB + "The maximum amount of CRYPTO frame data that can be buffered.") + +QUICHE_FLAG(bool, quic_allow_http3_priority, false, "If true, support HTTP/3 priority") + +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_max_aggressive_retransmittable_on_wire_ping_count, 0, + "If set to non-zero, the maximum number of consecutive pings that " + "can be sent with aggressive initial retransmittable on wire timeout " + "if there is no new data received. After which, the timeout will be " + "exponentially back off until exceeds the default ping timeout.") + +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_max_congestion_window, 2000, "The maximum congestion window in packets.") + QUICHE_FLAG(bool, http2_reloadable_flag_http2_testonly_default_false, false, "A testonly reloadable flag that will always default to false.") diff --git a/source/extensions/quic_listeners/quiche/platform/http2_ptr_util_impl.h b/source/extensions/quic_listeners/quiche/platform/http2_ptr_util_impl.h deleted file mode 100644 index 7ef0bd5828ae..000000000000 --- a/source/extensions/quic_listeners/quiche/platform/http2_ptr_util_impl.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include -#include - -// NOLINT(namespace-envoy) - -// This file is part of the QUICHE platform implementation, and is not to be -// consumed or referenced directly by other Envoy code. It serves purely as a -// porting layer for QUICHE. - -namespace http2 { - -template std::unique_ptr Http2MakeUniqueImpl(Args&&... args) { - return std::make_unique(std::forward(args)...); -} - -} // namespace http2 diff --git a/source/extensions/quic_listeners/quiche/platform/http2_string_impl.h b/source/extensions/quic_listeners/quiche/platform/http2_string_impl.h deleted file mode 100644 index fc54f8938f5c..000000000000 --- a/source/extensions/quic_listeners/quiche/platform/http2_string_impl.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include - -// NOLINT(namespace-envoy) - -// This file is part of the QUICHE platform implementation, and is not to be -// consumed or referenced directly by other Envoy code. It serves purely as a -// porting layer for QUICHE. - -namespace http2 { - -using Http2StringImpl = std::string; - -} // namespace http2 diff --git a/source/extensions/quic_listeners/quiche/platform/quic_export_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_export_impl.h index e3da61c839ec..afdcee72b8c5 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_export_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_export_impl.h @@ -8,3 +8,4 @@ #define QUIC_EXPORT #define QUIC_EXPORT_PRIVATE +#define QUIC_NO_EXPORT diff --git a/source/extensions/quic_listeners/quiche/platform/quic_mutex_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_mutex_impl.h index 4250eb45e227..15297625deef 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_mutex_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_mutex_impl.h @@ -12,6 +12,17 @@ namespace quic { +#define QUIC_EXCLUSIVE_LOCKS_REQUIRED_IMPL ABSL_EXCLUSIVE_LOCKS_REQUIRED +#define QUIC_GUARDED_BY_IMPL ABSL_GUARDED_BY +#define QUIC_LOCKABLE_IMPL ABSL_LOCKABLE +#define QUIC_LOCKS_EXCLUDED_IMPL ABSL_LOCKS_EXCLUDED +#define QUIC_SHARED_LOCKS_REQUIRED_IMPL ABSL_SHARED_LOCKS_REQUIRED +#define QUIC_EXCLUSIVE_LOCK_FUNCTION_IMPL ABSL_EXCLUSIVE_LOCK_FUNCTION +#define QUIC_UNLOCK_FUNCTION_IMPL ABSL_UNLOCK_FUNCTION +#define QUIC_SHARED_LOCK_FUNCTION_IMPL ABSL_SHARED_LOCK_FUNCTION +#define QUIC_SCOPED_LOCKABLE_IMPL ABSL_SCOPED_LOCKABLE +#define QUIC_ASSERT_SHARED_LOCK_IMPL ABSL_ASSERT_SHARED_LOCK + // A class wrapping a non-reentrant mutex. class LOCKABLE QUIC_EXPORT_PRIVATE QuicLockImpl { public: diff --git a/source/extensions/quic_listeners/quiche/platform/quic_string_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_string_impl.h deleted file mode 100644 index 9370a02b1eb0..000000000000 --- a/source/extensions/quic_listeners/quiche/platform/quic_string_impl.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include - -// NOLINT(namespace-envoy) - -// This file is part of the QUICHE platform implementation, and is not to be -// consumed or referenced directly by other Envoy code. It serves purely as a -// porting layer for QUICHE. - -namespace quic { - -using QuicStringImpl = std::string; - -} // namespace quic diff --git a/source/extensions/quic_listeners/quiche/platform/spdy_containers_impl.h b/source/extensions/quic_listeners/quiche/platform/spdy_containers_impl.h index 35d08c6183bf..d0205a46e362 100644 --- a/source/extensions/quic_listeners/quiche/platform/spdy_containers_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/spdy_containers_impl.h @@ -6,7 +6,6 @@ // consumed or referenced directly by other Envoy code. It serves purely as a // porting layer for QUICHE. -#include "extensions/quic_listeners/quiche/platform/spdy_string_impl.h" #include "extensions/quic_listeners/quiche/platform/spdy_string_piece_impl.h" #include "absl/container/flat_hash_map.h" diff --git a/source/extensions/quic_listeners/quiche/platform/spdy_string_impl.h b/source/extensions/quic_listeners/quiche/platform/spdy_string_impl.h deleted file mode 100644 index a1d5f9500d56..000000000000 --- a/source/extensions/quic_listeners/quiche/platform/spdy_string_impl.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include - -// NOLINT(namespace-envoy) - -// This file is part of the QUICHE platform implementation, and is not to be -// consumed or referenced directly by other Envoy code. It serves purely as a -// porting layer for QUICHE. - -namespace spdy { - -using SpdyStringImpl = std::string; - -} // namespace spdy diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc index 6171028b2269..87c6e320091c 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc @@ -295,7 +295,8 @@ TEST_P(EnvoyQuicServerSessionTest, ConnectionClose) { std::string error_details("dummy details"); quic::QuicErrorCode error(quic::QUIC_INVALID_FRAME_DATA); - quic::QuicConnectionCloseFrame frame(error, error_details); + quic::QuicConnectionCloseFrame frame(quic_version_[0].transport_version, error, error_details, + /* transport_close_frame_type = */ 0); EXPECT_CALL(network_connection_callbacks_, onEvent(Network::ConnectionEvent::RemoteClose)); quic_connection_->OnConnectionCloseFrame(frame); EXPECT_EQ(absl::StrCat(quic::QuicErrorCodeToString(error), " with details: ", error_details), diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc index 6137b87a159b..54304dd722db 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc @@ -44,7 +44,7 @@ class EnvoyQuicServerStreamTest : public testing::TestWithParam { *listener_config_.socket_), quic_session_(quic_config_, {quic_version_}, &quic_connection_, *dispatcher_, quic_config_.GetInitialStreamFlowControlWindowToSend() * 2), - stream_id_(VersionUsesQpack(quic_version_.transport_version) ? 4u : 5u), + stream_id_(VersionUsesHttp3(quic_version_.transport_version) ? 4u : 5u), quic_stream_(new EnvoyQuicServerStream(stream_id_, &quic_session_, quic::BIDIRECTIONAL)), response_headers_{{":status", "200"}} { quic_stream_->setDecoder(stream_decoder_); @@ -73,7 +73,7 @@ class EnvoyQuicServerStreamTest : public testing::TestWithParam { trailers_.OnHeaderBlockStart(); trailers_.OnHeader("key1", "value1"); - if (!quic::VersionUsesQpack(quic_version_.transport_version)) { + if (!quic::VersionUsesHttp3(quic_version_.transport_version)) { // ":final-offset" is required and stripped off by quic. trailers_.OnHeader(":final-offset", absl::StrCat("", request_body_.length())); } @@ -88,11 +88,10 @@ class EnvoyQuicServerStreamTest : public testing::TestWithParam { std::string bodyToStreamPayload(const std::string& body) { std::string data = body; - if (quic::VersionUsesQpack(quic_version_.transport_version)) { + if (quic::VersionUsesHttp3(quic_version_.transport_version)) { std::unique_ptr data_buffer; - quic::HttpEncoder encoder; quic::QuicByteCount data_frame_header_length = - encoder.SerializeDataFrameHeader(body.length(), &data_buffer); + quic::HttpEncoder::SerializeDataFrameHeader(body.length(), &data_buffer); quic::QuicStringPiece data_frame_header(data_buffer.get(), data_frame_header_length); data = absl::StrCat(data_frame_header, body); } @@ -107,7 +106,7 @@ class EnvoyQuicServerStreamTest : public testing::TestWithParam { EXPECT_EQ(Http::Headers::get().MethodValues.Post, headers->Method()->value().getStringView()); })); - if (quic::VersionUsesQpack(quic_version_.transport_version)) { + if (quic::VersionUsesHttp3(quic_version_.transport_version)) { quic_stream_->OnHeadersDecoded(request_headers_); } else { quic_stream_->OnStreamHeaderList(/*fin=*/false, request_headers_.uncompressed_header_bytes(), @@ -197,7 +196,7 @@ TEST_P(EnvoyQuicServerStreamTest, DecodeHeadersBodyAndTrailers) { TEST_P(EnvoyQuicServerStreamTest, OutOfOrderTrailers) { EXPECT_CALL(stream_callbacks_, onResetStream(_, _)); - if (quic::VersionUsesQpack(quic_version_.transport_version)) { + if (quic::VersionUsesHttp3(quic_version_.transport_version)) { return; } EXPECT_CALL(stream_decoder_, decodeHeaders_(_, /*end_stream=*/false)) @@ -277,7 +276,7 @@ TEST_P(EnvoyQuicServerStreamTest, ReadDisableAndReEnableImmediately) { EXPECT_EQ(Http::Headers::get().MethodValues.Post, headers->Method()->value().getStringView()); })); - if (quic::VersionUsesQpack(quic_version_.transport_version)) { + if (quic::VersionUsesHttp3(quic_version_.transport_version)) { quic_stream_->OnHeadersDecoded(request_headers_); } else { quic_stream_->OnStreamHeaderList(/*fin=*/false, request_headers_.uncompressed_header_bytes(), diff --git a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc index c11520f40022..b2848e0594db 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc @@ -668,7 +668,7 @@ TEST_F(FileUtilsTest, ReadFileContents) { } TEST_F(QuicPlatformTest, PickUnsedPort) { - int port = QuicPickUnusedPortOrDie(); + int port = QuicPickServerPortForTestsOrDie(); std::vector supported_versions = Envoy::TestEnvironment::getIpVersionsForTest(); for (auto ip_version : supported_versions) { @@ -694,7 +694,7 @@ TEST_F(QuicPlatformTest, FailToPickUnsedPort) { // Fail bind call's to mimic port exhaustion. EXPECT_CALL(os_sys_calls, bind(_, _, _)) .WillRepeatedly(Return(Envoy::Api::SysCallIntResult{-1, EADDRINUSE})); - EXPECT_DEATH_LOG_TO_STDERR(QuicPickUnusedPortOrDie(), "Failed to pick a port for test."); + EXPECT_DEATH_LOG_TO_STDERR(QuicPickServerPortForTestsOrDie(), "Failed to pick a port for test."); } TEST_F(QuicPlatformTest, TestEnvoyQuicBufferAllocator) { diff --git a/test/extensions/quic_listeners/quiche/platform/quic_port_utils_impl.cc b/test/extensions/quic_listeners/quiche/platform/quic_port_utils_impl.cc index 5dee83fb0420..de3f39a001a7 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_port_utils_impl.cc +++ b/test/extensions/quic_listeners/quiche/platform/quic_port_utils_impl.cc @@ -16,7 +16,7 @@ namespace quic { -int QuicPickUnusedPortOrDieImpl() { +int QuicPickServerPortForTestsOrDieImpl() { std::vector supported_versions = Envoy::TestEnvironment::getIpVersionsForTest(); ASSERT(!supported_versions.empty()); diff --git a/test/extensions/quic_listeners/quiche/platform/quic_port_utils_impl.h b/test/extensions/quic_listeners/quiche/platform/quic_port_utils_impl.h index 405266ca0c4e..61736db4b192 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_port_utils_impl.h +++ b/test/extensions/quic_listeners/quiche/platform/quic_port_utils_impl.h @@ -8,7 +8,7 @@ namespace quic { -int QuicPickUnusedPortOrDieImpl(); +int QuicPickServerPortForTestsOrDieImpl(); inline void QuicRecyclePortImpl(int) { // No-op with current port picking implementation. } From fb8235971792d924a429577f87cd86ea6cbc1a75 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 21 Nov 2019 23:45:14 -0800 Subject: [PATCH 16/29] build: several fix for C++17 compatibility (#9102) Risk Level: Low Testing: run asan test with --cxxopt=-std=c++17 Signed-off-by: Lizan Zhou --- include/envoy/http/header_map.h | 8 +- source/common/http/http1/header_formatter.cc | 4 +- .../common/aws/credentials_provider_impl.cc | 80 ++++++++++--------- .../common/aws/credentials_provider_impl.h | 12 +-- .../listener/tls_inspector/tls_inspector.cc | 3 +- test/common/http/http2/http2_frame.h | 2 +- .../aws/credentials_provider_impl_test.cc | 21 ++--- .../common/aws/credentials_provider_test.cc | 2 +- 8 files changed, 69 insertions(+), 63 deletions(-) diff --git a/include/envoy/http/header_map.h b/include/envoy/http/header_map.h index 893421d3c8f8..53703e1fd5b1 100644 --- a/include/envoy/http/header_map.h +++ b/include/envoy/http/header_map.h @@ -181,9 +181,13 @@ class HeaderString { */ Type type() const { return type_; } - bool operator==(const char* rhs) const { return getStringView() == absl::string_view(rhs); } + bool operator==(const char* rhs) const { + return getStringView() == absl::NullSafeStringView(rhs); + } bool operator==(absl::string_view rhs) const { return getStringView() == rhs; } - bool operator!=(const char* rhs) const { return getStringView() != absl::string_view(rhs); } + bool operator!=(const char* rhs) const { + return getStringView() != absl::NullSafeStringView(rhs); + } bool operator!=(absl::string_view rhs) const { return getStringView() != rhs; } private: diff --git a/source/common/http/http1/header_formatter.cc b/source/common/http/http1/header_formatter.cc index a33357029f9b..514f2383dff0 100644 --- a/source/common/http/http1/header_formatter.cc +++ b/source/common/http/http1/header_formatter.cc @@ -1,5 +1,7 @@ #include "common/http/http1/header_formatter.h" +#include + namespace Envoy { namespace Http { namespace Http1 { @@ -19,4 +21,4 @@ std::string ProperCaseHeaderKeyFormatter::format(absl::string_view key) const { } } // namespace Http1 } // namespace Http -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/source/extensions/filters/http/common/aws/credentials_provider_impl.cc b/source/extensions/filters/http/common/aws/credentials_provider_impl.cc index 43cde8c1ddd8..ba2de4e51050 100644 --- a/source/extensions/filters/http/common/aws/credentials_provider_impl.cc +++ b/source/extensions/filters/http/common/aws/credentials_provider_impl.cc @@ -12,43 +12,47 @@ namespace HttpFilters { namespace Common { namespace Aws { -constexpr static char AWS_ACCESS_KEY_ID[] = "AWS_ACCESS_KEY_ID"; -constexpr static char AWS_SECRET_ACCESS_KEY[] = "AWS_SECRET_ACCESS_KEY"; -constexpr static char AWS_SESSION_TOKEN[] = "AWS_SESSION_TOKEN"; - -constexpr static char ACCESS_KEY_ID[] = "AccessKeyId"; -constexpr static char SECRET_ACCESS_KEY[] = "SecretAccessKey"; -constexpr static char TOKEN[] = "Token"; -constexpr static char EXPIRATION[] = "Expiration"; -constexpr static char EXPIRATION_FORMAT[] = "%E4Y%m%dT%H%M%S%z"; -constexpr static char TRUE[] = "true"; - -constexpr static char AWS_CONTAINER_CREDENTIALS_RELATIVE_URI[] = - "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"; -constexpr static char AWS_CONTAINER_CREDENTIALS_FULL_URI[] = "AWS_CONTAINER_CREDENTIALS_FULL_URI"; -constexpr static char AWS_CONTAINER_AUTHORIZATION_TOKEN[] = "AWS_CONTAINER_AUTHORIZATION_TOKEN"; -constexpr static char AWS_EC2_METADATA_DISABLED[] = "AWS_EC2_METADATA_DISABLED"; - -constexpr static std::chrono::hours REFRESH_INTERVAL{1}; -constexpr static std::chrono::seconds REFRESH_GRACE_PERIOD{5}; -constexpr static char EC2_METADATA_HOST[] = "169.254.169.254:80"; -constexpr static char CONTAINER_METADATA_HOST[] = "169.254.170.2:80"; -constexpr static char SECURITY_CREDENTIALS_PATH[] = "/latest/meta-data/iam/security-credentials"; +namespace { + +constexpr char AWS_ACCESS_KEY_ID[] = "AWS_ACCESS_KEY_ID"; +constexpr char AWS_SECRET_ACCESS_KEY[] = "AWS_SECRET_ACCESS_KEY"; +constexpr char AWS_SESSION_TOKEN[] = "AWS_SESSION_TOKEN"; + +constexpr char ACCESS_KEY_ID[] = "AccessKeyId"; +constexpr char SECRET_ACCESS_KEY[] = "SecretAccessKey"; +constexpr char TOKEN[] = "Token"; +constexpr char EXPIRATION[] = "Expiration"; +constexpr char EXPIRATION_FORMAT[] = "%E4Y%m%dT%H%M%S%z"; +constexpr char TRUE[] = "true"; + +constexpr char AWS_CONTAINER_CREDENTIALS_RELATIVE_URI[] = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"; +constexpr char AWS_CONTAINER_CREDENTIALS_FULL_URI[] = "AWS_CONTAINER_CREDENTIALS_FULL_URI"; +constexpr char AWS_CONTAINER_AUTHORIZATION_TOKEN[] = "AWS_CONTAINER_AUTHORIZATION_TOKEN"; +constexpr char AWS_EC2_METADATA_DISABLED[] = "AWS_EC2_METADATA_DISABLED"; + +constexpr std::chrono::hours REFRESH_INTERVAL{1}; +constexpr std::chrono::seconds REFRESH_GRACE_PERIOD{5}; +constexpr char EC2_METADATA_HOST[] = "169.254.169.254:80"; +constexpr char CONTAINER_METADATA_HOST[] = "169.254.170.2:80"; +constexpr char SECURITY_CREDENTIALS_PATH[] = "/latest/meta-data/iam/security-credentials"; + +} // namespace Credentials EnvironmentCredentialsProvider::getCredentials() { ENVOY_LOG(debug, "Getting AWS credentials from the environment"); - const auto access_key_id = std::getenv(AWS_ACCESS_KEY_ID); - if (access_key_id == nullptr) { + const auto access_key_id = absl::NullSafeStringView(std::getenv(AWS_ACCESS_KEY_ID)); + if (access_key_id.empty()) { return Credentials(); } - const auto secret_access_key = std::getenv(AWS_SECRET_ACCESS_KEY); - const auto session_token = std::getenv(AWS_SESSION_TOKEN); + const auto secret_access_key = absl::NullSafeStringView(std::getenv(AWS_SECRET_ACCESS_KEY)); + const auto session_token = absl::NullSafeStringView(std::getenv(AWS_SESSION_TOKEN)); ENVOY_LOG(debug, "Found following AWS credentials in the environment: {}={}, {}={}, {}={}", - AWS_ACCESS_KEY_ID, access_key_id ? access_key_id : "", AWS_SECRET_ACCESS_KEY, - secret_access_key ? "*****" : "", AWS_SESSION_TOKEN, session_token ? "*****" : ""); + AWS_ACCESS_KEY_ID, access_key_id, AWS_SECRET_ACCESS_KEY, + secret_access_key.empty() ? "" : "*****", AWS_SESSION_TOKEN, + session_token.empty() ? "" : "*****"); return Credentials(access_key_id, secret_access_key, session_token); } @@ -186,17 +190,19 @@ DefaultCredentialsProviderChain::DefaultCredentialsProviderChain( ENVOY_LOG(debug, "Using environment credentials provider"); add(factories.createEnvironmentCredentialsProvider()); - const auto relative_uri = std::getenv(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI); - const auto full_uri = std::getenv(AWS_CONTAINER_CREDENTIALS_FULL_URI); - const auto metadata_disabled = std::getenv(AWS_EC2_METADATA_DISABLED); + const auto relative_uri = + absl::NullSafeStringView(std::getenv(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI)); + const auto full_uri = absl::NullSafeStringView(std::getenv(AWS_CONTAINER_CREDENTIALS_FULL_URI)); + const auto metadata_disabled = absl::NullSafeStringView(std::getenv(AWS_EC2_METADATA_DISABLED)); - if (relative_uri != nullptr) { - const auto uri = std::string(CONTAINER_METADATA_HOST) + relative_uri; + if (!relative_uri.empty()) { + const auto uri = absl::StrCat(CONTAINER_METADATA_HOST, relative_uri); ENVOY_LOG(debug, "Using task role credentials provider with URI: {}", uri); add(factories.createTaskRoleCredentialsProvider(api, metadata_fetcher, uri)); - } else if (full_uri != nullptr) { - const auto authorization_token = std::getenv(AWS_CONTAINER_AUTHORIZATION_TOKEN); - if (authorization_token != nullptr) { + } else if (!full_uri.empty()) { + const auto authorization_token = + absl::NullSafeStringView(std::getenv(AWS_CONTAINER_AUTHORIZATION_TOKEN)); + if (!authorization_token.empty()) { ENVOY_LOG(debug, "Using task role credentials provider with URI: " "{} and authorization token", @@ -207,7 +213,7 @@ DefaultCredentialsProviderChain::DefaultCredentialsProviderChain( ENVOY_LOG(debug, "Using task role credentials provider with URI: {}", full_uri); add(factories.createTaskRoleCredentialsProvider(api, metadata_fetcher, full_uri)); } - } else if (metadata_disabled == nullptr || strncmp(metadata_disabled, TRUE, strlen(TRUE)) != 0) { + } else if (metadata_disabled != TRUE) { ENVOY_LOG(debug, "Using instance profile credentials provider"); add(factories.createInstanceProfileCredentialsProvider(api, metadata_fetcher)); } diff --git a/source/extensions/filters/http/common/aws/credentials_provider_impl.h b/source/extensions/filters/http/common/aws/credentials_provider_impl.h index 5f9b3816cfe0..36a0067e596c 100644 --- a/source/extensions/filters/http/common/aws/credentials_provider_impl.h +++ b/source/extensions/filters/http/common/aws/credentials_provider_impl.h @@ -10,6 +10,8 @@ #include "extensions/filters/http/common/aws/credentials_provider.h" +#include "absl/strings/string_view.h" + namespace Envoy { namespace Extensions { namespace HttpFilters { @@ -78,8 +80,8 @@ class InstanceProfileCredentialsProvider : public MetadataCredentialsProviderBas class TaskRoleCredentialsProvider : public MetadataCredentialsProviderBase { public: TaskRoleCredentialsProvider(Api::Api& api, const MetadataFetcher& metadata_fetcher, - const std::string& credential_uri, - const std::string& authorization_token = std::string()) + absl::string_view credential_uri, + absl::string_view authorization_token = {}) : MetadataCredentialsProviderBase(api, metadata_fetcher), credential_uri_(credential_uri), authorization_token_(authorization_token) {} @@ -118,8 +120,7 @@ class CredentialsProviderChainFactories { virtual CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, - const std::string& credential_uri, - const std::string& authorization_token = std::string()) const PURE; + absl::string_view credential_uri, absl::string_view authorization_token = {}) const PURE; virtual CredentialsProviderSharedPtr createInstanceProfileCredentialsProvider( Api::Api& api, @@ -150,8 +151,7 @@ class DefaultCredentialsProviderChain : public CredentialsProviderChain, CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, - const std::string& credential_uri, - const std::string& authorization_token = std::string()) const override { + absl::string_view credential_uri, absl::string_view authorization_token = {}) const override { return std::make_shared(api, metadata_fetcher, credential_uri, authorization_token); } diff --git a/source/extensions/filters/listener/tls_inspector/tls_inspector.cc b/source/extensions/filters/listener/tls_inspector/tls_inspector.cc index af7f79869bca..e744e1fe506a 100644 --- a/source/extensions/filters/listener/tls_inspector/tls_inspector.cc +++ b/source/extensions/filters/listener/tls_inspector/tls_inspector.cc @@ -48,7 +48,8 @@ Config::Config(Stats::Scope& scope, uint32_t max_client_hello_size) SSL_CTX_set_tlsext_servername_callback( ssl_ctx_.get(), [](SSL* ssl, int* out_alert, void*) -> int { Filter* filter = static_cast(SSL_get_app_data(ssl)); - filter->onServername(SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name)); + filter->onServername( + absl::NullSafeStringView(SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name))); // Return an error to stop the handshake; we have what we wanted already. *out_alert = SSL_AD_USER_CANCELLED; diff --git a/test/common/http/http2/http2_frame.h b/test/common/http/http2/http2_frame.h index b39d465ba4c6..9b04779a0a44 100644 --- a/test/common/http/http2/http2_frame.h +++ b/test/common/http/http2/http2_frame.h @@ -69,7 +69,7 @@ class Http2Frame { enum class ResponseStatus { Unknown, Ok, NotFound }; // Methods for creating HTTP2 frames - static Http2Frame makePingFrame(absl::string_view data = nullptr); + static Http2Frame makePingFrame(absl::string_view data = {}); static Http2Frame makeEmptySettingsFrame(SettingsFlags flags = SettingsFlags::None); static Http2Frame makeEmptyHeadersFrame(uint32_t stream_index, HeadersFlags flags = HeadersFlags::None); diff --git a/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc b/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc index b72ea27af87f..f595cb6dfc1f 100644 --- a/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc +++ b/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc @@ -340,21 +340,14 @@ class DefaultCredentialsProviderChainTest : public testing::Test { class MockCredentialsProviderChainFactories : public CredentialsProviderChainFactories { public: MOCK_CONST_METHOD0(createEnvironmentCredentialsProvider, CredentialsProviderSharedPtr()); - MOCK_CONST_METHOD4(createTaskRoleCredentialsProviderMock, + MOCK_CONST_METHOD4(createTaskRoleCredentialsProvider, CredentialsProviderSharedPtr( Api::Api&, const MetadataCredentialsProviderBase::MetadataFetcher&, - const std::string&, const std::string&)); + absl::string_view, absl::string_view)); MOCK_CONST_METHOD2(createInstanceProfileCredentialsProvider, CredentialsProviderSharedPtr( Api::Api&, const MetadataCredentialsProviderBase::MetadataFetcher& fetcher)); - - CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( - Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, - const std::string& credential_uri, const std::string& authorization_token) const override { - return createTaskRoleCredentialsProviderMock(api, metadata_fetcher, credential_uri, - authorization_token); - } }; Event::SimulatedTimeSystem time_system_; @@ -381,22 +374,22 @@ TEST_F(DefaultCredentialsProviderChainTest, MetadataNotDisabled) { TEST_F(DefaultCredentialsProviderChainTest, RelativeUri) { TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/path/to/creds", 1); - EXPECT_CALL(factories_, createTaskRoleCredentialsProviderMock( - Ref(*api_), _, "169.254.170.2:80/path/to/creds", "")); + EXPECT_CALL(factories_, createTaskRoleCredentialsProvider(Ref(*api_), _, + "169.254.170.2:80/path/to/creds", "")); DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); } TEST_F(DefaultCredentialsProviderChainTest, FullUriNoAuthorizationToken) { TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://host/path/to/creds", 1); - EXPECT_CALL(factories_, createTaskRoleCredentialsProviderMock(Ref(*api_), _, - "http://host/path/to/creds", "")); + EXPECT_CALL(factories_, + createTaskRoleCredentialsProvider(Ref(*api_), _, "http://host/path/to/creds", "")); DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); } TEST_F(DefaultCredentialsProviderChainTest, FullUriWithAuthorizationToken) { TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://host/path/to/creds", 1); TestEnvironment::setEnvVar("AWS_CONTAINER_AUTHORIZATION_TOKEN", "auth_token", 1); - EXPECT_CALL(factories_, createTaskRoleCredentialsProviderMock( + EXPECT_CALL(factories_, createTaskRoleCredentialsProvider( Ref(*api_), _, "http://host/path/to/creds", "auth_token")); DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); } diff --git a/test/extensions/filters/http/common/aws/credentials_provider_test.cc b/test/extensions/filters/http/common/aws/credentials_provider_test.cc index a492ec9b2a87..32c9546ed666 100644 --- a/test/extensions/filters/http/common/aws/credentials_provider_test.cc +++ b/test/extensions/filters/http/common/aws/credentials_provider_test.cc @@ -16,7 +16,7 @@ TEST(Credentials, Default) { } TEST(Credentials, AllNull) { - const auto c = Credentials(nullptr, nullptr, nullptr); + const auto c = Credentials({}, {}, {}); EXPECT_FALSE(c.accessKeyId().has_value()); EXPECT_FALSE(c.secretAccessKey().has_value()); EXPECT_FALSE(c.sessionToken().has_value()); From 40970bd6edb9969a59cd4ed17d7c66b2c0dcb8ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Fri, 22 Nov 2019 14:00:00 -0300 Subject: [PATCH 17/29] thrift-proxy: fix crash (#9089) After Router::onEvent() handles a local or remote close, do not attempt to close the connection a second time during Router::onDestroy(). Risk Level: low Testing: unit/integration tests Doc Changes: n/a Release Notes: n/a Fixes: #9037 Signed-off-by: Raul Gutierrez Segales --- .../thrift_proxy/router/router_impl.cc | 28 ++++++++++++---- .../network/thrift_proxy/router/router_impl.h | 1 + .../network/thrift_proxy/integration_test.cc | 32 +++++++++++++++++++ .../network/thrift_proxy/router_test.cc | 21 ++++++++++++ 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc index ef501d7ba0a2..449fddb85f63 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc @@ -184,8 +184,8 @@ RouteConstSharedPtr RouteMatcher::route(const MessageMetadata& metadata, void Router::onDestroy() { if (upstream_request_ != nullptr) { upstream_request_->resetStream(); + cleanup(); } - cleanup(); } void Router::setDecoderFilterCallbacks(ThriftFilters::DecoderFilterCallbacks& callbacks) { @@ -354,10 +354,12 @@ void Router::onEvent(Network::ConnectionEvent event) { switch (event) { case Network::ConnectionEvent::RemoteClose: + ENVOY_STREAM_LOG(debug, "upstream remote close", *callbacks_); upstream_request_->onResetStream( Tcp::ConnectionPool::PoolFailureReason::RemoteConnectionFailure); break; case Network::ConnectionEvent::LocalClose: + ENVOY_STREAM_LOG(debug, "upstream local close", *callbacks_); upstream_request_->onResetStream( Tcp::ConnectionPool::PoolFailureReason::LocalConnectionFailure); break; @@ -365,6 +367,8 @@ void Router::onEvent(Network::ConnectionEvent event) { // Connected is consumed by the connection pool. NOT_REACHED_GCOVR_EXCL_LINE; } + + upstream_request_->releaseConnection(false); } const Network::Connection* Router::downstreamConnection() const { @@ -389,7 +393,11 @@ Router::UpstreamRequest::UpstreamRequest(Router& parent, Tcp::ConnectionPool::In protocol_(NamedProtocolConfigFactory::getFactory(protocol_type).createProtocol()), request_complete_(false), response_started_(false), response_complete_(false) {} -Router::UpstreamRequest::~UpstreamRequest() = default; +Router::UpstreamRequest::~UpstreamRequest() { + if (conn_pool_handle_) { + conn_pool_handle_->cancel(Tcp::ConnectionPool::CancelPolicy::Default); + } +} FilterStatus Router::UpstreamRequest::start() { Tcp::ConnectionPool::Cancellable* handle = conn_pool_.newConnection(*this); @@ -407,18 +415,24 @@ FilterStatus Router::UpstreamRequest::start() { return FilterStatus::Continue; } -void Router::UpstreamRequest::resetStream() { +void Router::UpstreamRequest::releaseConnection(const bool close) { if (conn_pool_handle_) { conn_pool_handle_->cancel(Tcp::ConnectionPool::CancelPolicy::Default); + conn_pool_handle_ = nullptr; } - if (conn_data_ != nullptr) { - conn_state_ = nullptr; - conn_data_->connection().close(Network::ConnectionCloseType::NoFlush); - conn_data_.reset(); + conn_state_ = nullptr; + + // The event triggered by close will also release this connection so clear conn_data_ before + // closing. + auto conn_data = std::move(conn_data_); + if (close && conn_data != nullptr) { + conn_data->connection().close(Network::ConnectionCloseType::NoFlush); } } +void Router::UpstreamRequest::resetStream() { releaseConnection(true); } + void Router::UpstreamRequest::onPoolFailure(Tcp::ConnectionPool::PoolFailureReason reason, Upstream::HostDescriptionConstSharedPtr host) { conn_pool_handle_ = nullptr; diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.h b/source/extensions/filters/network/thrift_proxy/router/router_impl.h index 47cd9e929ecf..d61c7fe7b59a 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.h +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.h @@ -206,6 +206,7 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, FilterStatus start(); void resetStream(); + void releaseConnection(bool close); // Tcp::ConnectionPool::Callbacks void onPoolFailure(Tcp::ConnectionPool::PoolFailureReason reason, diff --git a/test/extensions/filters/network/thrift_proxy/integration_test.cc b/test/extensions/filters/network/thrift_proxy/integration_test.cc index 2e3f4dfe96b8..23f23c663d9e 100644 --- a/test/extensions/filters/network/thrift_proxy/integration_test.cc +++ b/test/extensions/filters/network/thrift_proxy/integration_test.cc @@ -7,6 +7,7 @@ #include "gtest/gtest.h" using testing::Combine; +using testing::HasSubstr; using ::testing::TestParamInfo; using testing::Values; @@ -301,6 +302,37 @@ TEST_P(ThriftConnManagerIntegrationTest, EarlyCloseWithUpstream) { EXPECT_EQ(1U, counter->value()); } +// Regression test for https://github.com/envoyproxy/envoy/issues/9037. +TEST_P(ThriftConnManagerIntegrationTest, EarlyUpstreamClose) { + initializeCall(DriverMode::Success); + + const std::string partial_request = + request_bytes_.toString().substr(0, request_bytes_.length() - 5); + + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("listener_0")); + tcp_client->write(request_bytes_.toString()); + + FakeUpstream* expected_upstream = getExpectedUpstream(false); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(expected_upstream->waitForRawConnection(fake_upstream_connection)); + + std::string data; + ASSERT_TRUE(fake_upstream_connection->waitForData(request_bytes_.length(), &data)); + Buffer::OwnedImpl upstream_request(data); + EXPECT_EQ(request_bytes_.toString(), upstream_request.toString()); + + ASSERT_TRUE(fake_upstream_connection->close()); + + tcp_client->waitForDisconnect(); + + EXPECT_THAT(tcp_client->data(), HasSubstr("connection failure")); + + Stats::CounterSharedPtr counter = test_server_->counter("thrift.thrift_stats.request_call"); + EXPECT_EQ(1U, counter->value()); + counter = test_server_->counter("thrift.thrift_stats.response_exception"); + EXPECT_EQ(1U, counter->value()); +} + TEST_P(ThriftConnManagerIntegrationTest, Oneway) { initializeOneway(); diff --git a/test/extensions/filters/network/thrift_proxy/router_test.cc b/test/extensions/filters/network/thrift_proxy/router_test.cc index 457bbafdc4e1..5b29318b4997 100644 --- a/test/extensions/filters/network/thrift_proxy/router_test.cc +++ b/test/extensions/filters/network/thrift_proxy/router_test.cc @@ -603,6 +603,27 @@ TEST_F(ThriftRouterTest, UnexpectedUpstreamLocalClose) { router_->onEvent(Network::ConnectionEvent::RemoteClose); } +// Regression test for https://github.com/envoyproxy/envoy/issues/9037. +TEST_F(ThriftRouterTest, DontCloseConnectionTwice) { + initializeRouter(); + startRequest(MessageType::Call); + connectUpstream(); + sendTrivialStruct(FieldType::String); + + EXPECT_CALL(callbacks_, sendLocalReply(_, _)) + .WillOnce(Invoke([&](const DirectResponse& response, bool end_stream) -> void { + auto& app_ex = dynamic_cast(response); + EXPECT_EQ(AppExceptionType::InternalError, app_ex.type_); + EXPECT_THAT(app_ex.what(), ContainsRegex(".*connection failure.*")); + EXPECT_TRUE(end_stream); + })); + router_->onEvent(Network::ConnectionEvent::RemoteClose); + + // Connection close shouldn't happen in onDestroy(), since it's been handled. + EXPECT_CALL(upstream_connection_, close(Network::ConnectionCloseType::NoFlush)).Times(0); + destroyRouter(); +} + TEST_F(ThriftRouterTest, UnexpectedRouterDestroyBeforeUpstreamConnect) { initializeRouter(); startRequest(MessageType::Call); From cddf4acdd5fcd09cc54d829cbe3e5dfd0b36d511 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Fri, 22 Nov 2019 09:45:43 -0800 Subject: [PATCH 18/29] test flake: fix header casing integration test (#9104) Given that most other integration tests don't set a timeout for an upstream connection that is expected to succeed, remove the timeout. Risk Level: low, test-only Testing: 1000 repetitions Doc Changes: n/a Release Notes: n/a Fixes: #8899 Signed-off-by: Stephan Zuercher --- test/integration/header_casing_integration_test.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/integration/header_casing_integration_test.cc b/test/integration/header_casing_integration_test.cc index e5181b26880b..78104d71541d 100644 --- a/test/integration/header_casing_integration_test.cc +++ b/test/integration/header_casing_integration_test.cc @@ -1,5 +1,3 @@ -#include - #include "envoy/config/bootstrap/v2/bootstrap.pb.h" #include "common/buffer/buffer_impl.h" @@ -51,8 +49,7 @@ TEST_P(HeaderCasingIntegrationTest, VerifyCasedHeaders) { tcp_client->write(request, false); Envoy::FakeRawConnectionPtr upstream_connection; - ASSERT_TRUE( - fake_upstreams_[0]->waitForRawConnection(upstream_connection, std::chrono::milliseconds(10))); + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(upstream_connection)); // Verify that the upstream request has proper cased headers. std::string upstream_request; @@ -74,4 +71,4 @@ TEST_P(HeaderCasingIntegrationTest, VerifyCasedHeaders) { tcp_client->close(); } -} // namespace Envoy \ No newline at end of file +} // namespace Envoy From 76f172f7a92ce8af27909d93fc6a05b61d909029 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 22 Nov 2019 23:37:50 -0800 Subject: [PATCH 19/29] build: bazel 1.2.0 (#9108) Description: Risk Level: Low Testing: CI Docs Changes: Release Notes: Signed-off-by: Lizan Zhou --- .bazelrc | 4 ++-- .bazelversion | 2 +- bazel/repository_locations.bzl | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.bazelrc b/.bazelrc index aa01e30bf3d4..9661c979514d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -133,7 +133,7 @@ build:remote --spawn_strategy=remote,sandboxed,local build:remote --strategy=Javac=remote,sandboxed,local build:remote --strategy=Closure=remote,sandboxed,local build:remote --strategy=Genrule=remote,sandboxed,local -build:remote --remote_timeout=3600 +build:remote --remote_timeout=7200 build:remote --auth_enabled=true build:remote --remote_download_toplevel @@ -148,7 +148,7 @@ build:remote-gcc --config=rbe-toolchain-gcc # Docker sandbox # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L7 -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu@sha256:3ca8acc35fdb57ab26e1bb5f9488f37095f45acd77a12602510410dbefa00b58 +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu@sha256:f0b2453c3587e3297f5caf5e97fbf57c97592c96136209ec13fe2795aae2c896 build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker diff --git a/.bazelversion b/.bazelversion index 9084fa2f716a..26aaba0e8663 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -1.1.0 +1.2.0 diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index e916d616dc5b..b47007af1ea6 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -9,11 +9,11 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.0/bazel-gazelle-v0.19.0.tar.gz"], ), bazel_toolchains = dict( - sha256 = "1e16833a9f0e32b292568c0dfee7bd48133c2038605757d3a430551394310006", - strip_prefix = "bazel-toolchains-1.1.0", + sha256 = "83352b6e68fa797184071f35e3b67c7c8815efadcea81bb9cdb6bbbf2e07d389", + strip_prefix = "bazel-toolchains-1.1.3", urls = [ - "https://github.com/bazelbuild/bazel-toolchains/releases/download/1.1.0/bazel-toolchains-1.1.0.tar.gz", - "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/1.1.0.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/releases/download/1.1.3/bazel-toolchains-1.1.3.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/1.1.3.tar.gz", ], ), build_bazel_rules_apple = dict( @@ -22,9 +22,9 @@ REPOSITORY_LOCATIONS = dict( sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e", ), envoy_build_tools = dict( - sha256 = "d0f88bef8bd7f76c3684407977f5673f3d06a6c50d4ddaffb8f0e7df6b0ef69e", - strip_prefix = "envoy-build-tools-a6b28555badcb18d6be924c8fc1bea49971656b8", - urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/a6b28555badcb18d6be924c8fc1bea49971656b8.tar.gz"], + sha256 = "a81ff3a12adedfc4641a926c9b167c53bea62784a81ac9ced7893436c709b60b", + strip_prefix = "envoy-build-tools-07314d549e27e9a4033af6236888d2a9ee0ad443", + urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/07314d549e27e9a4033af6236888d2a9ee0ad443.tar.gz"], ), boringssl = dict( sha256 = "3eea198c8e3f587ffc8ea6acf87d7575f571bbe6dd88ec90405e236303f3dc01", From 4cd080d11b0a5d76d9f9d7b2bd5523a976ee39b2 Mon Sep 17 00:00:00 2001 From: l8huang Date: Sun, 24 Nov 2019 12:10:20 -0800 Subject: [PATCH 20/29] fix unstable test case UdpListenerImplTest.SendData (#9119) If Network::Test::readFromSocket() return error code 'Again', `data.addresses_.peer_` is null, `data.addresses_.peer_->asString()` causes crash because it dereferences a null pointer. Signed-off-by: lhuang8 --- test/common/network/udp_listener_impl_test.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/common/network/udp_listener_impl_test.cc b/test/common/network/udp_listener_impl_test.cc index 062f0ebc933b..94408da5242e 100644 --- a/test/common/network/udp_listener_impl_test.cc +++ b/test/common/network/udp_listener_impl_test.cc @@ -411,8 +411,6 @@ TEST_P(UdpListenerImplTest, SendData) { client_socket_->ioHandle(), *client_socket_->localAddress(), data); bytes_read = result.rc_; - EXPECT_EQ(send_from_addr->asString(), data.addresses_.peer_->asString()); - if (bytes_read >= bytes_to_read || retry == 10 || result.err_->getErrorCode() != Api::IoError::IoErrorCode::Again) { break; @@ -423,6 +421,7 @@ TEST_P(UdpListenerImplTest, SendData) { ASSERT(bytes_read == 0); } while (true); EXPECT_EQ(bytes_to_read, bytes_read); + EXPECT_EQ(send_from_addr->asString(), data.addresses_.peer_->asString()); EXPECT_EQ(data.buffer_->toString(), payload); } From 44a12a1f298f5c60eb251f7ab774a0e9812c90df Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Sun, 24 Nov 2019 12:10:38 -0800 Subject: [PATCH 21/29] buffer: misc missed evbuffer references (#9117) Signed-off-by: Matt Klein --- include/envoy/buffer/buffer.h | 7 ------- source/common/event/libevent.h | 12 ------------ .../quiche/platform/quic_mem_slice_storage_impl.cc | 7 +++---- 3 files changed, 3 insertions(+), 23 deletions(-) diff --git a/include/envoy/buffer/buffer.h b/include/envoy/buffer/buffer.h index 937d0a30faea..116369f3cf41 100644 --- a/include/envoy/buffer/buffer.h +++ b/include/envoy/buffer/buffer.h @@ -130,13 +130,6 @@ class Instance { * @return the actual number of slices needed, which may be greater than out_size. Passing * nullptr for out and 0 for out_size will just return the size of the array needed * to capture all of the slice data. - * TODO(mattklein123): WARNING: The underlying implementation of this function currently uses - * libevent's evbuffer. It has the infuriating property where calling getRawSlices(nullptr, 0) - * will return the slices that include all of the buffer data, but not any empty slices at the - * end. However, calling getRawSlices(iovec, SOME_CONST), WILL return potentially empty slices - * beyond the end of the buffer. Code that is trying to avoid stack overflow by limiting the - * number of returned slices needs to deal with this. When we get rid of evbuffer we can rework - * all of this. */ virtual uint64_t getRawSlices(RawSlice* out, uint64_t out_size) const PURE; diff --git a/source/common/event/libevent.h b/source/common/event/libevent.h index 20fdfefcbcd6..dd7f7cd25ebb 100644 --- a/source/common/event/libevent.h +++ b/source/common/event/libevent.h @@ -7,16 +7,6 @@ extern "C" { void event_base_free(event_base*); } -struct evbuffer; -extern "C" { -void evbuffer_free(evbuffer*); -} - -struct bufferevent; -extern "C" { -void bufferevent_free(bufferevent*); -} - struct evconnlistener; extern "C" { void evconnlistener_free(evconnlistener*); @@ -44,8 +34,6 @@ class Global { }; using BasePtr = CSmartPtr; -using BufferPtr = CSmartPtr; -using BufferEventPtr = CSmartPtr; using ListenerPtr = CSmartPtr; } // namespace Libevent diff --git a/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_storage_impl.cc b/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_storage_impl.cc index 53d83156447a..7522c9178124 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_storage_impl.cc +++ b/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_storage_impl.cc @@ -31,10 +31,9 @@ QuicMemSliceStorageImpl::QuicMemSliceStorageImpl(const struct iovec* iov, int io while (io_offset < write_len) { size_t slice_len = std::min(write_len - io_offset, max_slice_len); Envoy::Buffer::RawSlice slice; - // Populate a temporary buffer instance and then move it to |buffer_|. - // This is necessary for the old evbuffer implementation of OwnedImpl where - // consecutive reserve/commit can return addresses in same slice which - // violates the restriction of |max_slice_len| when ToSpan() is called. + // Populate a temporary buffer instance and then move it to |buffer_|. This is necessary because + // consecutive reserve/commit can return addresses in same slice which violates the restriction + // of |max_slice_len| when ToSpan() is called. Envoy::Buffer::OwnedImpl buffer; uint16_t num_slice = buffer.reserve(slice_len, &slice, 1); ASSERT(num_slice == 1); From 883a074f67815872831776cd56586a9ec172bd7c Mon Sep 17 00:00:00 2001 From: Bo Shi Date: Sun, 24 Nov 2019 15:11:46 -0500 Subject: [PATCH 22/29] docs: reword least request from N random load balancing link. (#9113) https://github.com/envoyproxy/data-plane-api/pull/529 Signed-off-by: Bo Shi --- .../arch_overview/upstream/load_balancing/load_balancers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst b/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst index 5b6c4bb5c40f..48605a7ad6d1 100644 --- a/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst +++ b/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst @@ -33,7 +33,7 @@ same or different weights. * *all weights equal*: An O(1) algorithm which selects N random available hosts as specified in the :ref:`configuration ` (2 by default) and picks the - host which has the fewest active requests (`Research + host which has the fewest active requests (`Mitzenmacher et al. `_ has shown that this approach is nearly as good as an O(N) full scan). This is also known as P2C (power of two choices). The P2C load balancer has the property that a host with the highest number of active From bb76ead628035142490a852ccace2cdbd50fe0db Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Sun, 24 Nov 2019 12:16:06 -0800 Subject: [PATCH 23/29] test flake: fix proxy protocol test (#9103) The client and listener events on connection are not strictly ordered (at least not on macOS), so make sure that all expected events fire before triggering the dispatcher exit. Risk Level: low, test-only change Testing: 2000x iterations, no failures Docs Changes: n/a Release Notes: n/a Signed-off-by: Stephan Zuercher --- .../listener/proxy_protocol/proxy_protocol_test.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc index 5ef3ebce1d0c..0724ceccae64 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc @@ -93,10 +93,19 @@ class ProxyProtocolTest : public testing::TestWithParam void { + expected_callbacks--; + if (expected_callbacks == 0) { + dispatcher_->exit(); + } + }; + EXPECT_CALL(factory_, createListenerFilterChain(_)) .WillOnce(Invoke([&](Network::ListenerFilterManager& filter_manager) -> bool { filter_manager.addAcceptFilter( std::make_unique(std::make_shared(listenerScope()))); + maybeExitDispatcher(); return true; })); conn_->connect(); @@ -112,7 +121,7 @@ class ProxyProtocolTest : public testing::TestWithParam void { dispatcher_->exit(); })); + .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { maybeExitDispatcher(); })); dispatcher_->run(Event::Dispatcher::RunType::Block); } From aa1fc310eb25e24005f4d14e221e30c53a7fd2fc Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Sun, 24 Nov 2019 15:44:09 -0500 Subject: [PATCH 24/29] upstream: reject invalid config when using zone aware routing (#9031) Adds checks to reject config that configures endpoint with a pirority > 0 when using zone aware load balancing (e.g. when a local cluster is defined). Fixes #9013 Signed-off-by: Snow Pettersen --- include/envoy/upstream/cluster_manager.h | 5 +- source/common/upstream/BUILD | 2 + .../common/upstream/cluster_manager_impl.cc | 21 ++--- source/common/upstream/cluster_manager_impl.h | 8 +- source/common/upstream/eds.cc | 9 +-- source/common/upstream/eds.h | 1 - source/common/upstream/logical_dns_cluster.cc | 25 +++++- source/common/upstream/static_cluster.cc | 3 + source/common/upstream/strict_dns_cluster.cc | 4 + source/common/upstream/upstream_impl.cc | 10 +++ source/common/upstream/upstream_impl.h | 5 ++ .../upstream/cluster_manager_impl_test.cc | 77 +++++++++++++++++++ test/common/upstream/eds_test.cc | 6 +- .../clusters/custom_static_cluster.h | 2 +- test/integration/stats_integration_test.cc | 6 +- test/mocks/upstream/mocks.h | 4 +- 16 files changed, 157 insertions(+), 31 deletions(-) diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 8974edf4cd08..d7c52af32baf 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -195,9 +195,10 @@ class ClusterManager { /** * Return the local cluster name, if it was configured. * - * @return std::string the local cluster name, or "" if no local cluster was configured. + * @return absl::optional the local cluster name, or empty if no local cluster was + * configured. */ - virtual const std::string& localClusterName() const PURE; + virtual const absl::optional& localClusterName() const PURE; /** * This method allows to register callbacks for cluster lifecycle events in the ClusterManager. diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index aaec1e74f06f..b44d89ed75c8 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -234,6 +234,7 @@ envoy_cc_library( "//source/common/protobuf", "//source/common/protobuf:utility_lib", "//source/extensions/clusters:well_known_names", + "@envoy_api//envoy/api/v2:pkg_cc_proto", ], ) @@ -461,6 +462,7 @@ envoy_cc_library( "//source/common/stats:isolated_store_lib", "//source/common/stats:stats_lib", "//source/server:transport_socket_config_lib", + "@envoy_api//envoy/api/v2:pkg_cc_proto", "@envoy_api//envoy/api/v2/core:pkg_cc_proto", "@envoy_api//envoy/api/v2/endpoint:pkg_cc_proto", ], diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index da694ab7ddaf..f3241d7dacab 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -220,6 +220,12 @@ ClusterManagerImpl::ClusterManagerImpl( } } + // We need to know whether we're zone aware early on, so make sure we do this lookup + // before we load any clusters. + if (!cm_config.local_cluster_name().empty()) { + local_cluster_name_ = cm_config.local_cluster_name(); + } + const auto& dyn_resources = bootstrap.dynamic_resources(); // Cluster loading happens in two phases: first all the primary clusters are loaded, and then all @@ -288,19 +294,15 @@ ClusterManagerImpl::ClusterManagerImpl( cm_stats_.cluster_added_.add(bootstrap.static_resources().clusters().size()); updateClusterCounts(); - absl::optional local_cluster_name; - if (!cm_config.local_cluster_name().empty()) { - local_cluster_name_ = cm_config.local_cluster_name(); - local_cluster_name = cm_config.local_cluster_name(); - if (active_clusters_.find(local_cluster_name.value()) == active_clusters_.end()) { - throw EnvoyException( - fmt::format("local cluster '{}' must be defined", local_cluster_name.value())); - } + if (local_cluster_name_ && + (active_clusters_.find(local_cluster_name_.value()) == active_clusters_.end())) { + throw EnvoyException( + fmt::format("local cluster '{}' must be defined", local_cluster_name_.value())); } // Once the initial set of static bootstrap clusters are created (including the local cluster), // we can instantiate the thread local cluster manager. - tls_->set([this, local_cluster_name]( + tls_->set([this, local_cluster_name = local_cluster_name_]( Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(*this, dispatcher, local_cluster_name); }); @@ -1060,7 +1062,6 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::updateClusterMembership( const std::string& name, uint32_t priority, PrioritySet::UpdateHostsParams update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, ThreadLocal::Slot& tls, uint64_t overprovisioning_factor) { - ThreadLocalClusterManagerImpl& config = tls.getTyped(); ASSERT(config.thread_local_clusters_.find(name) != config.thread_local_clusters_.end()); diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 4740267a29d3..e4d8830925a2 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -229,7 +229,9 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable& localClusterName() const override { + return local_cluster_name_; + } ClusterUpdateCallbacksHandlePtr addThreadLocalClusterUpdateCallbacks(ClusterUpdateCallbacks&) override; @@ -472,8 +474,8 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable local_cluster_name_; Grpc::AsyncClientManagerPtr async_client_manager_; Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; TimeSource& time_source_; diff --git a/source/common/upstream/eds.cc b/source/common/upstream/eds.cc index 2f6d96d4cfe8..5851445763c1 100644 --- a/source/common/upstream/eds.cc +++ b/source/common/upstream/eds.cc @@ -13,7 +13,7 @@ EdsClusterImpl::EdsClusterImpl( Stats::ScopePtr&& stats_scope, bool added_via_api) : BaseDynamicClusterImpl(cluster, runtime, factory_context, std::move(stats_scope), added_via_api), - cm_(factory_context.clusterManager()), local_info_(factory_context.localInfo()), + local_info_(factory_context.localInfo()), cluster_name_(cluster.eds_cluster_config().service_name().empty() ? cluster.name() : cluster.eds_cluster_config().service_name()), @@ -41,13 +41,8 @@ void EdsClusterImpl::BatchUpdateHelper::batchUpdate(PrioritySet::HostUpdateCb& h std::unordered_map updated_hosts; PriorityStateManager priority_state_manager(parent_, parent_.local_info_, &host_update_cb); for (const auto& locality_lb_endpoint : cluster_load_assignment_.endpoints()) { - const uint32_t priority = locality_lb_endpoint.priority(); + parent_.validateEndpointsForZoneAwareRouting(locality_lb_endpoint); - if (priority > 0 && !parent_.cluster_name_.empty() && - parent_.cluster_name_ == parent_.cm_.localClusterName()) { - throw EnvoyException(fmt::format("Unexpected non-zero priority for local cluster '{}'.", - parent_.cluster_name_)); - } priority_state_manager.initializePriorityFor(locality_lb_endpoint); for (const auto& lb_endpoint : locality_lb_endpoint.lb_endpoints()) { diff --git a/source/common/upstream/eds.h b/source/common/upstream/eds.h index ec8fc3d1ef33..a19859add5b7 100644 --- a/source/common/upstream/eds.h +++ b/source/common/upstream/eds.h @@ -69,7 +69,6 @@ class EdsClusterImpl : public BaseDynamicClusterImpl, Config::SubscriptionCallba const envoy::api::v2::ClusterLoadAssignment& cluster_load_assignment_; }; - const ClusterManager& cm_; std::unique_ptr subscription_; const LocalInfo::LocalInfo& local_info_; const std::string cluster_name_; diff --git a/source/common/upstream/logical_dns_cluster.cc b/source/common/upstream/logical_dns_cluster.cc index f9096ee19426..bf73083e0e35 100644 --- a/source/common/upstream/logical_dns_cluster.cc +++ b/source/common/upstream/logical_dns_cluster.cc @@ -5,6 +5,8 @@ #include #include +#include "envoy/api/v2/eds.pb.h" +#include "envoy/common/exception.h" #include "envoy/stats/scope.h" #include "common/common/fmt.h" @@ -17,6 +19,27 @@ namespace Envoy { namespace Upstream { +namespace { +envoy::api::v2::ClusterLoadAssignment +convertPriority(const envoy::api::v2::ClusterLoadAssignment& load_assignment) { + envoy::api::v2::ClusterLoadAssignment converted; + converted.MergeFrom(load_assignment); + + // We convert the priority set by the configuration back to zero. This helps + // ensure that we don't blow up later on when using zone aware routing due + // to a check that all priorities are zero. + // + // Since LOGICAL_DNS is limited to exactly one host declared per load_assignment + // (checked in the ctor in this file), we can safely just rewrite the priority + // to zero. + for (auto& endpoint : *converted.mutable_endpoints()) { + endpoint.set_priority(0); + } + + return converted; +} +} // namespace + LogicalDnsCluster::LogicalDnsCluster( const envoy::api::v2::Cluster& cluster, Runtime::Loader& runtime, Network::DnsResolverSharedPtr dns_resolver, @@ -31,7 +54,7 @@ LogicalDnsCluster::LogicalDnsCluster( factory_context.dispatcher().createTimer([this]() -> void { startResolve(); })), local_info_(factory_context.localInfo()), load_assignment_(cluster.has_load_assignment() - ? cluster.load_assignment() + ? convertPriority(cluster.load_assignment()) : Config::Utility::translateClusterHosts(cluster.hosts())) { failure_backoff_strategy_ = Config::Utility::prepareDnsRefreshStrategy( cluster, dns_refresh_rate_ms_.count(), factory_context.random()); diff --git a/source/common/upstream/static_cluster.cc b/source/common/upstream/static_cluster.cc index 9ed6044d990d..37feab2ea41a 100644 --- a/source/common/upstream/static_cluster.cc +++ b/source/common/upstream/static_cluster.cc @@ -1,5 +1,7 @@ #include "common/upstream/static_cluster.h" +#include "envoy/common/exception.h" + namespace Envoy { namespace Upstream { @@ -19,6 +21,7 @@ StaticClusterImpl::StaticClusterImpl( cluster_load_assignment.policy(), overprovisioning_factor, kDefaultOverProvisioningFactor); for (const auto& locality_lb_endpoint : cluster_load_assignment.endpoints()) { + validateEndpointsForZoneAwareRouting(locality_lb_endpoint); priority_state_manager_->initializePriorityFor(locality_lb_endpoint); for (const auto& lb_endpoint : locality_lb_endpoint.lb_endpoints()) { priority_state_manager_->registerHostForPriority( diff --git a/source/common/upstream/strict_dns_cluster.cc b/source/common/upstream/strict_dns_cluster.cc index 49cdd1d50885..a676059857f4 100644 --- a/source/common/upstream/strict_dns_cluster.cc +++ b/source/common/upstream/strict_dns_cluster.cc @@ -1,5 +1,7 @@ #include "common/upstream/strict_dns_cluster.h" +#include "envoy/common/exception.h" + namespace Envoy { namespace Upstream { @@ -23,6 +25,8 @@ StrictDnsClusterImpl::StrictDnsClusterImpl( : Config::Utility::translateClusterHosts(cluster.hosts())); const auto& locality_lb_endpoints = load_assignment.endpoints(); for (const auto& locality_lb_endpoint : locality_lb_endpoints) { + validateEndpointsForZoneAwareRouting(locality_lb_endpoint); + for (const auto& lb_endpoint : locality_lb_endpoint.lb_endpoints()) { const auto& socket_address = lb_endpoint.endpoint().address().socket_address(); if (!socket_address.resolver_name().empty()) { diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index f18b9c17a16a..cac0fb841af3 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -817,6 +817,8 @@ ClusterImplBase::ClusterImplBase( Stats::ScopePtr&& stats_scope, bool added_via_api) : init_manager_(fmt::format("Cluster {}", cluster.name())), init_watcher_("ClusterImplBase", [this]() { onInitDone(); }), runtime_(runtime), + local_cluster_(factory_context.clusterManager().localClusterName().value_or("") == + cluster.name()), symbol_table_(stats_scope->symbolTable()) { factory_context.setInitManager(init_manager_); auto socket_factory = createTransportSocketFactory(cluster, factory_context); @@ -1016,6 +1018,14 @@ ClusterImplBase::resolveProtoAddress(const envoy::api::v2::core::Address& addres } } +void ClusterImplBase::validateEndpointsForZoneAwareRouting( + const envoy::api::v2::endpoint::LocalityLbEndpoints& endpoints) const { + if (local_cluster_ && endpoints.priority() > 0) { + throw EnvoyException( + fmt::format("Unexpected non-zero priority for local cluster '{}'.", info()->name())); + } +} + ClusterInfoImpl::ResourceManagers::ResourceManagers(const envoy::api::v2::Cluster& config, Runtime::Loader& runtime, const std::string& cluster_name, diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 4a438594b529..c5f50b3540df 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -12,6 +12,7 @@ #include #include "envoy/api/v2/core/base.pb.h" +#include "envoy/api/v2/eds.pb.h" #include "envoy/api/v2/endpoint/endpoint.pb.h" #include "envoy/config/typed_metadata.h" #include "envoy/event/timer.h" @@ -736,6 +737,9 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable initialization_complete_callback_; uint64_t pending_initialize_health_checks_{}; + const bool local_cluster_; Stats::SymbolTable& symbol_table_; }; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 091e4fbc529e..7798cfc69c8d 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -3690,6 +3690,83 @@ TEST_F(ClusterManagerImplTest, ConnPoolsNotDrainedOnHostSetChange) { hosts_added, {}, 100); } +TEST_F(ClusterManagerImplTest, InvalidPriorityLocalClusterNameStatic) { + std::string yaml = R"EOF( +static_resources: + clusters: + - name: new_cluster + connect_timeout: 4s + type: STATIC + load_assignment: + cluster_name: "domains" + endpoints: + - priority: 10 + lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.2 + port_value: 11001 +cluster_manager: + local_cluster_name: new_cluster +)EOF"; + + EXPECT_THROW_WITH_MESSAGE(create(parseBootstrapFromV2Yaml(yaml)), EnvoyException, + "Unexpected non-zero priority for local cluster 'new_cluster'."); +} + +TEST_F(ClusterManagerImplTest, InvalidPriorityLocalClusterNameStrictDns) { + std::string yaml = R"EOF( +static_resources: + clusters: + - name: new_cluster + connect_timeout: 4s + type: STRICT_DNS + load_assignment: + cluster_name: "domains" + endpoints: + - priority: 10 + lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.2 + port_value: 11001 +cluster_manager: + local_cluster_name: new_cluster +)EOF"; + + EXPECT_THROW_WITH_MESSAGE(create(parseBootstrapFromV2Yaml(yaml)), EnvoyException, + "Unexpected non-zero priority for local cluster 'new_cluster'."); +} + +TEST_F(ClusterManagerImplTest, InvalidPriorityLocalClusterNameLogicalDns) { + std::string yaml = R"EOF( +static_resources: + clusters: + - name: new_cluster + connect_timeout: 4s + type: LOGICAL_DNS + load_assignment: + cluster_name: "domains" + endpoints: + - priority: 10 + lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.2 + port_value: 11001 +cluster_manager: + local_cluster_name: new_cluster +)EOF"; + + // The priority for LOGICAL_DNS endpoints are written, so we just verify that there is only a + // single priority even if the endpoint was configured to be priority 10. + create(parseBootstrapFromV2Yaml(yaml)); + const auto cluster = cluster_manager_->get("new_cluster"); + EXPECT_EQ(1, cluster->prioritySet().hostSetsPerPriority().size()); +} } // namespace } // namespace Upstream } // namespace Envoy diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index d66a7c27223a..377471ff61ad 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -1371,9 +1371,11 @@ TEST_F(EdsTest, EndpointHostsPerPriority) { // Make sure config updates with P!=0 are rejected for the local cluster. TEST_F(EdsTest, NoPriorityForLocalCluster) { + cm_.local_cluster_name_ = "name"; + resetCluster(); + envoy::api::v2::ClusterLoadAssignment cluster_load_assignment; cluster_load_assignment.set_cluster_name("fare"); - cm_.local_cluster_name_ = "fare"; uint32_t port = 1000; auto add_hosts_to_priority = [&cluster_load_assignment, &port](uint32_t priority, uint32_t n) { auto* endpoints = cluster_load_assignment.add_endpoints(); @@ -1397,7 +1399,7 @@ TEST_F(EdsTest, NoPriorityForLocalCluster) { Protobuf::RepeatedPtrField resources; resources.Add()->PackFrom(cluster_load_assignment); EXPECT_THROW_WITH_MESSAGE(eds_callbacks_->onConfigUpdate(resources, ""), EnvoyException, - "Unexpected non-zero priority for local cluster 'fare'."); + "Unexpected non-zero priority for local cluster 'name'."); // Try an update which only has endpoints with P=0. This should go through. cluster_load_assignment.clear_endpoints(); diff --git a/test/integration/clusters/custom_static_cluster.h b/test/integration/clusters/custom_static_cluster.h index f3e9d3fd7c09..77c3aa671b1d 100644 --- a/test/integration/clusters/custom_static_cluster.h +++ b/test/integration/clusters/custom_static_cluster.h @@ -112,4 +112,4 @@ class CustomStaticClusterFactoryWithLb : public CustomStaticClusterFactoryBase { : CustomStaticClusterFactoryBase("envoy.clusters.custom_static_with_lb", true) {} }; -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index 73b2303af4a6..52af564948ed 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -259,6 +259,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // 2019/10/17 8484 43340 44000 stats: add unit support to histogram // 2019/11/01 8859 43563 44000 build: switch to libc++ by default // 2019/11/15 9040 43371 44000 build: update protobuf to 3.10.1 + // 2019/11/15 9040 43403 35500 upstream: track whether cluster is local // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -272,7 +273,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // If you encounter a failure here, please see // https://github.com/envoyproxy/envoy/blob/master/source/docs/stats.md#stats-memory-tests // for details on how to fix. - EXPECT_MEMORY_EQ(m_per_cluster, 43371); // 104 bytes higher than a debug build. + EXPECT_MEMORY_EQ(m_per_cluster, 43403); // 104 bytes higher than a debug build. EXPECT_MEMORY_LE(m_per_cluster, 44000); } @@ -305,6 +306,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // 2019/10/17 8484 34998 35000 stats: add unit support to histogram // 2019/11/01 8859 35221 36000 build: switch to libc++ by default // 2019/11/15 9040 35029 35500 build: update protobuf to 3.10.1 + // 2019/11/15 9040 35061 35500 upstream: track whether cluster is local // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -318,7 +320,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // If you encounter a failure here, please see // https://github.com/envoyproxy/envoy/blob/master/source/docs/stats.md#stats-memory-tests // for details on how to fix. - EXPECT_MEMORY_EQ(m_per_cluster, 35029); // 104 bytes higher than a debug build. + EXPECT_MEMORY_EQ(m_per_cluster, 35061); // 104 bytes higher than a debug build. EXPECT_MEMORY_LE(m_per_cluster, 35500); } diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 4b7151d295d8..1e4f9fd6360e 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -324,7 +324,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD0(adsMux, Config::GrpcMuxSharedPtr()); MOCK_METHOD0(grpcAsyncClientManager, Grpc::AsyncClientManager&()); MOCK_CONST_METHOD0(versionInfo, const std::string()); - MOCK_CONST_METHOD0(localClusterName, const std::string&()); + MOCK_CONST_METHOD0(localClusterName, const absl::optional&()); MOCK_METHOD1(addThreadLocalClusterUpdateCallbacks_, ClusterUpdateCallbacksHandle*(ClusterUpdateCallbacks& callbacks)); MOCK_METHOD0(subscriptionFactory, Config::SubscriptionFactory&()); @@ -336,7 +336,7 @@ class MockClusterManager : public ClusterManager { envoy::api::v2::core::BindConfig bind_config_; std::shared_ptr> ads_mux_; NiceMock async_client_manager_; - std::string local_cluster_name_; + absl::optional local_cluster_name_; NiceMock cluster_manager_factory_; NiceMock subscription_factory_; }; From 00ac6a166f7c28b7840e5af4cf9ba7b33a49eea5 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Mon, 25 Nov 2019 23:17:28 +0530 Subject: [PATCH 25/29] hot restart: send used stats only during hot restart (#9121) Signed-off-by: Rama Chavali --- source/server/hot_restarting_parent.cc | 17 +++++++++++------ test/server/hot_restarting_parent_test.cc | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/source/server/hot_restarting_parent.cc b/source/server/hot_restarting_parent.cc index 553f446aec0b..d42e8960fbc2 100644 --- a/source/server/hot_restarting_parent.cc +++ b/source/server/hot_restarting_parent.cc @@ -116,14 +116,19 @@ HotRestartingParent::Internal::getListenSocketsForChild(const HotRestartMessage: // names. The problem can be solved by splitting the export up over many chunks. void HotRestartingParent::Internal::exportStatsToChild(HotRestartMessage::Reply::Stats* stats) { for (const auto& gauge : server_->stats().gauges()) { - (*stats->mutable_gauges())[gauge->name()] = gauge->value(); + if (gauge->used()) { + (*stats->mutable_gauges())[gauge->name()] = gauge->value(); + } } + for (const auto& counter : server_->stats().counters()) { - // The hot restart parent is expected to have stopped its normal stat exporting (and so - // latching) by the time it begins exporting to the hot restart child. - uint64_t latched_value = counter->latch(); - if (latched_value > 0) { - (*stats->mutable_counter_deltas())[counter->name()] = latched_value; + if (counter->used()) { + // The hot restart parent is expected to have stopped its normal stat exporting (and so + // latching) by the time it begins exporting to the hot restart child. + uint64_t latched_value = counter->latch(); + if (latched_value > 0) { + (*stats->mutable_counter_deltas())[counter->name()] = latched_value; + } } } stats->set_memory_allocated(Memory::Stats::totalCurrentlyAllocated()); diff --git a/test/server/hot_restarting_parent_test.cc b/test/server/hot_restarting_parent_test.cc index 076c525a4a93..0f77074baa2a 100644 --- a/test/server/hot_restarting_parent_test.cc +++ b/test/server/hot_restarting_parent_test.cc @@ -94,6 +94,20 @@ TEST_F(HotRestartingParentTest, exportStatsToChild) { EXPECT_EQ(124, stats.gauges().at("g1")); EXPECT_EQ(455, stats.gauges().at("g2")); } + + // When a counter and gauge are not used, they should not be included in the message. + { + store.counter("unused_counter"); + store.counter("used_counter").inc(); + store.gauge("unused_gauge", Stats::Gauge::ImportMode::Accumulate); + store.gauge("used_gauge", Stats::Gauge::ImportMode::Accumulate).add(1); + HotRestartMessage::Reply::Stats stats; + hot_restarting_parent_.exportStatsToChild(&stats); + EXPECT_EQ(stats.counter_deltas().end(), stats.counter_deltas().find("unused_counter")); + EXPECT_EQ(1, stats.counter_deltas().at("used_counter")); + EXPECT_EQ(stats.gauges().end(), stats.counter_deltas().find("unused_gauge")); + EXPECT_EQ(1, stats.gauges().at("used_gauge")); + } } TEST_F(HotRestartingParentTest, drainListeners) { From be0d183f61d076a37060aa421e4ccc54d47963c1 Mon Sep 17 00:00:00 2001 From: Tony Allen Date: Mon, 25 Nov 2019 09:51:48 -0800 Subject: [PATCH 26/29] adaptive concurrency: Add buffer to minRTT measurements (#9038) Signed-off-by: Tony Allen --- .../v2alpha/adaptive_concurrency.proto | 12 +- .../v3alpha/adaptive_concurrency.proto | 12 +- .../adaptive_concurrency_filter.rst | 16 ++- .../gradient_controller.cc | 22 ++-- .../gradient_controller.h | 29 +++-- .../gradient_controller_test.cc | 105 +++++++++--------- 6 files changed, 109 insertions(+), 87 deletions(-) diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto index 0972192a95dc..a4ddb8819eff 100644 --- a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto +++ b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto @@ -25,10 +25,6 @@ message GradientControllerConfig { // Parameters controlling the periodic recalculation of the concurrency limit from sampled request // latencies. message ConcurrencyLimitCalculationParams { - // The maximum value the gradient is allowed to take. This influences how aggressively the - // concurrency limit can increase. Defaults to 2.0. - google.protobuf.DoubleValue max_gradient = 1 [(validate.rules).double = {gt: 1.0}]; - // The allowed upper-bound on the calculated concurrency limit. Defaults to 1000. google.protobuf.UInt32Value max_concurrency_limit = 2 [(validate.rules).uint32 = {gt: 0}]; @@ -40,6 +36,7 @@ message GradientControllerConfig { } // Parameters controlling the periodic minRTT recalculation. + // [#next-free-field: 6] message MinimumRTTCalculationParams { // The time interval between recalculating the minimum request round-trip time. google.protobuf.Duration interval = 1 [(validate.rules).duration = { @@ -60,6 +57,13 @@ message GradientControllerConfig { // The concurrency limit set while measuring the minRTT. Defaults to 3. google.protobuf.UInt32Value min_concurrency = 4 [(validate.rules).uint32 = {gt: 0}]; + + // Amount added to the measured minRTT to add stability to the concurrency limit during natural + // variability in latency. This is expressed as a percentage of the measured value and can be + // adjusted to allow more or less tolerance to the sampled latency values. + // + // Defaults to 25%. + type.Percent buffer = 5; } // The percentile to use when summarizing aggregated samples. Defaults to p50. diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto index 5b4323fe0ebe..eb9d8791ad54 100644 --- a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto +++ b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto @@ -25,10 +25,6 @@ message GradientControllerConfig { // Parameters controlling the periodic recalculation of the concurrency limit from sampled request // latencies. message ConcurrencyLimitCalculationParams { - // The maximum value the gradient is allowed to take. This influences how aggressively the - // concurrency limit can increase. Defaults to 2.0. - google.protobuf.DoubleValue max_gradient = 1 [(validate.rules).double = {gt: 1.0}]; - // The allowed upper-bound on the calculated concurrency limit. Defaults to 1000. google.protobuf.UInt32Value max_concurrency_limit = 2 [(validate.rules).uint32 = {gt: 0}]; @@ -40,6 +36,7 @@ message GradientControllerConfig { } // Parameters controlling the periodic minRTT recalculation. + // [#next-free-field: 6] message MinimumRTTCalculationParams { // The time interval between recalculating the minimum request round-trip time. google.protobuf.Duration interval = 1 [(validate.rules).duration = { @@ -60,6 +57,13 @@ message GradientControllerConfig { // The concurrency limit set while measuring the minRTT. Defaults to 3. google.protobuf.UInt32Value min_concurrency = 4 [(validate.rules).uint32 = {gt: 0}]; + + // Amount added to the measured minRTT to add stability to the concurrency limit during natural + // variability in latency. This is expressed as a percentage of the measured value and can be + // adjusted to allow more or less tolerance to the sampled latency values. + // + // Defaults to 25%. + type.v3alpha.Percent buffer = 5; } // The percentile to use when summarizing aggregated samples. Defaults to p50. diff --git a/docs/root/configuration/http/http_filters/adaptive_concurrency_filter.rst b/docs/root/configuration/http/http_filters/adaptive_concurrency_filter.rst index 8addb23e5d88..1b75d78ec250 100644 --- a/docs/root/configuration/http/http_filters/adaptive_concurrency_filter.rst +++ b/docs/root/configuration/http/http_filters/adaptive_concurrency_filter.rst @@ -64,9 +64,19 @@ The gradient is calculated using summarized sampled request latencies (sampleRTT .. math:: - gradient = \frac{minRTT}{sampleRTT} + gradient = \frac{minRTT + B}{sampleRTT} This gradient value has a useful property, such that it decreases as the sampled latencies increase. +Notice that *B*, the buffer value added to the minRTT, allows for normal variance in the sampled +latencies by requiring the sampled latencies the exceed the minRTT by some configurable threshold +before decreasing the gradient value. + +The buffer will be a percentage of the measured minRTT value whose value is modified via the buffer field in the :ref:`minRTT calculation parameters `. The buffer is calculated as follows: + +.. math:: + + B = minRTT * buffer_{pct} + The gradient value is then used to update the concurrency limit via: .. math:: @@ -163,8 +173,8 @@ adaptive_concurrency.gradient_controller.sample_rtt_calc_interval_ms adaptive_concurrency.gradient_controller.max_concurrency_limit Overrides the maximum allowed concurrency limit. -adaptive_concurrency.gradient_controller.max_gradient - Overrides the maximum allowed gradient value. +adaptive_concurrency.gradient_controller.min_rtt_buffer + Overrides the padding added to the minRTT when calculating the concurrency limit. adaptive_concurrency.gradient_controller.sample_aggregate_percentile Overrides the percentile value used to represent the collection of latency samples in diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc index 7e4205d71d23..95f54aa2d5fc 100644 --- a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc @@ -37,13 +37,12 @@ GradientControllerConfig::GradientControllerConfig( proto_config.concurrency_limit_params(), max_concurrency_limit, 1000)), min_rtt_aggregate_request_count_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config.min_rtt_calc_params(), request_count, 50)), - max_gradient_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config.concurrency_limit_params(), - max_gradient, 2.0)), sample_aggregate_percentile_( PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(proto_config, sample_aggregate_percentile, 50)), - min_concurrency_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config.min_rtt_calc_params(), - min_concurrency, 3)) {} - + min_concurrency_( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config.min_rtt_calc_params(), min_concurrency, 3)), + min_rtt_buffer_pct_( + PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(proto_config.min_rtt_calc_params(), buffer, 25)) {} GradientController::GradientController(GradientControllerConfig config, Event::Dispatcher& dispatcher, Runtime::Loader&, const std::string& stats_prefix, Stats::Scope& scope, @@ -148,10 +147,14 @@ std::chrono::microseconds GradientController::processLatencySamplesAndClear() { } uint32_t GradientController::calculateNewLimit() { - // Calculate the gradient value, ensuring it remains below the configured maximum. ASSERT(sample_rtt_.count() > 0); - const double raw_gradient = static_cast(min_rtt_.count()) / sample_rtt_.count(); - const double gradient = std::min(config_.maxGradient(), raw_gradient); + + // Calculate the gradient value, ensuring it's clamped between 0.5 and 2.0. + // This prevents extreme changes in the concurrency limit between each sample + // window. + const auto buffered_min_rtt = min_rtt_.count() + min_rtt_.count() * config_.minRTTBufferPercent(); + const double raw_gradient = static_cast(buffered_min_rtt) / sample_rtt_.count(); + const double gradient = std::max(0.5, std::min(2.0, raw_gradient)); stats_.gradient_.set(gradient); const double limit = concurrencyLimit() * gradient; @@ -160,9 +163,8 @@ uint32_t GradientController::calculateNewLimit() { // The final concurrency value factors in the burst headroom and must be clamped to keep the value // in the range [1, configured_max]. - const auto clamp = [](int min, int max, int val) { return std::max(min, std::min(max, val)); }; const uint32_t new_limit = limit + burst_headroom; - return clamp(1, config_.maxConcurrencyLimit(), new_limit); + return std::max(1, std::min(config_.maxConcurrencyLimit(), new_limit)); } RequestForwardingAction GradientController::forwardingDecision() { diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h index a41da7201622..6cd64144901c 100644 --- a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h @@ -70,11 +70,6 @@ class GradientControllerConfig : public Logger::Loggable { min_rtt_aggregate_request_count_); } - double maxGradient() const { - return std::max( - 1.0, runtime_.snapshot().getDouble(RuntimeKeys::get().MaxGradientKey, max_gradient_)); - } - // The percentage is normalized to the range [0.0, 1.0]. double sampleAggregatePercentile() const { const double val = runtime_.snapshot().getDouble( @@ -82,7 +77,7 @@ class GradientControllerConfig : public Logger::Loggable { return std::max(0.0, std::min(val, 100.0)) / 100.0; } - // The percentage is normalized and clamped to the range [0.0, 1.0]. + // The percentage is normalized to the range [0.0, 1.0]. double jitterPercent() const { const double val = runtime_.snapshot().getDouble(RuntimeKeys::get().JitterPercentKey, jitter_pct_); @@ -93,6 +88,13 @@ class GradientControllerConfig : public Logger::Loggable { return runtime_.snapshot().getInteger(RuntimeKeys::get().MinConcurrencyKey, min_concurrency_); } + // The percentage is normalized to the range [0.0, 1.0]. + double minRTTBufferPercent() const { + const double val = runtime_.snapshot().getDouble(RuntimeKeys::get().MinRTTBufferPercentKey, + min_rtt_buffer_pct_); + return std::max(0.0, std::min(val, 100.0)) / 100.0; + } + private: class RuntimeKeyValues { public: @@ -104,12 +106,13 @@ class GradientControllerConfig : public Logger::Loggable { "adaptive_concurrency.gradient_controller.max_concurrency_limit"; const std::string MinRTTAggregateRequestCountKey = "adaptive_concurrency.gradient_controller.min_rtt_aggregate_request_count"; - const std::string MaxGradientKey = "adaptive_concurrency.gradient_controller.max_gradient"; const std::string SampleAggregatePercentileKey = "adaptive_concurrency.gradient_controller.sample_aggregate_percentile"; const std::string JitterPercentKey = "adaptive_concurrency.gradient_controller.jitter"; const std::string MinConcurrencyKey = "adaptive_concurrency.gradient_controller.min_concurrency"; + const std::string MinRTTBufferPercentKey = + "adaptive_concurrency.gradient_controller.min_rtt_buffer"; }; using RuntimeKeys = ConstSingleton; @@ -131,14 +134,14 @@ class GradientControllerConfig : public Logger::Loggable { // The number of requests to aggregate/sample during the minRTT recalculation. const uint32_t min_rtt_aggregate_request_count_; - // The maximum value the gradient may take. - const double max_gradient_; - // The percentile value considered when processing samples. const double sample_aggregate_percentile_; // The concurrency limit set while measuring the minRTT. const uint32_t min_concurrency_; + + // The amount added to the measured minRTT as a hedge against natural variability in latency. + const double min_rtt_buffer_pct_; }; using GradientControllerConfigSharedPtr = std::shared_ptr; @@ -248,9 +251,11 @@ class GradientController : public ConcurrencyController { // is non-zero, then we are actively in the minRTT sampling window. std::atomic deferred_limit_value_; - // Stores the expected upstream latency value under ideal conditions. This is the numerator in the - // gradient value explained above. + // Stores the expected upstream latency value under ideal conditions with the added buffer to + // account for variable latencies. This is the numerator in the gradient value. std::chrono::nanoseconds min_rtt_; + + // Stores the aggregated sampled latencies for use in the gradient calculation. std::chrono::nanoseconds sample_rtt_ ABSL_GUARDED_BY(sample_mutation_mtx_); // Tracks the count of requests that have been forwarded whose replies have diff --git a/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc b/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc index 7753ff474603..c5413bddba31 100644 --- a/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc +++ b/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc @@ -88,7 +88,6 @@ TEST_F(GradientControllerConfigTest, BasicTest) { sample_aggregate_percentile: value: 42.5 concurrency_limit_params: - max_gradient: 2.1 max_concurrency_limit: 1337 concurrency_update_interval: 0.123s min_rtt_calc_params: @@ -105,7 +104,6 @@ TEST_F(GradientControllerConfigTest, BasicTest) { EXPECT_EQ(config.sampleRTTCalcInterval(), std::chrono::milliseconds(123)); EXPECT_EQ(config.maxConcurrencyLimit(), 1337); EXPECT_EQ(config.minRTTAggregateRequestCount(), 52); - EXPECT_EQ(config.maxGradient(), 2.1); EXPECT_EQ(config.sampleAggregatePercentile(), .425); EXPECT_EQ(config.jitterPercent(), .132); EXPECT_EQ(config.minConcurrency(), 8); @@ -116,7 +114,6 @@ TEST_F(GradientControllerConfigTest, Clamping) { sample_aggregate_percentile: value: 42.5 concurrency_limit_params: - max_gradient: 2.1 max_concurrency_limit: 1337 concurrency_update_interval: nanos: 123000000 @@ -148,11 +145,12 @@ TEST_F(GradientControllerConfigTest, BasicTestOverrides) { sample_aggregate_percentile: value: 42.5 concurrency_limit_params: - max_gradient: 2.1 max_concurrency_limit: 1337 concurrency_update_interval: nanos: 123000000 min_rtt_calc_params: + buffer: + value: 33 jitter: value: 13.2 interval: @@ -175,9 +173,6 @@ TEST_F(GradientControllerConfigTest, BasicTestOverrides) { EXPECT_CALL(runtime_.snapshot_, getInteger(_, 52)).WillOnce(Return(65)); EXPECT_EQ(config.minRTTAggregateRequestCount(), 65); - EXPECT_CALL(runtime_.snapshot_, getDouble(_, 2.1)).WillOnce(Return(12.3)); - EXPECT_EQ(config.maxGradient(), 12.3); - EXPECT_CALL(runtime_.snapshot_, getDouble(_, 42.5)).WillOnce(Return(66.0)); EXPECT_EQ(config.sampleAggregatePercentile(), .66); @@ -186,6 +181,9 @@ TEST_F(GradientControllerConfigTest, BasicTestOverrides) { EXPECT_CALL(runtime_.snapshot_, getInteger(_, 7)).WillOnce(Return(9)); EXPECT_EQ(config.minConcurrency(), 9); + + EXPECT_CALL(runtime_.snapshot_, getDouble(_, 33.0)).WillOnce(Return(77.0)); + EXPECT_EQ(config.minRTTBufferPercent(), .77); } TEST_F(GradientControllerConfigTest, DefaultValuesTest) { @@ -202,10 +200,10 @@ TEST_F(GradientControllerConfigTest, DefaultValuesTest) { EXPECT_EQ(config.sampleRTTCalcInterval(), std::chrono::milliseconds(123)); EXPECT_EQ(config.maxConcurrencyLimit(), 1000); EXPECT_EQ(config.minRTTAggregateRequestCount(), 50); - EXPECT_EQ(config.maxGradient(), 2.0); EXPECT_EQ(config.sampleAggregatePercentile(), .5); EXPECT_EQ(config.jitterPercent(), .15); EXPECT_EQ(config.minConcurrency(), 3); + EXPECT_EQ(config.minRTTBufferPercent(), 0.25); } TEST_F(GradientControllerTest, MinRTTLogicTest) { @@ -213,7 +211,6 @@ TEST_F(GradientControllerTest, MinRTTLogicTest) { sample_aggregate_percentile: value: 50 concurrency_limit_params: - max_gradient: 2.0 max_concurrency_limit: concurrency_update_interval: 0.1s min_rtt_calc_params: @@ -261,7 +258,6 @@ TEST_F(GradientControllerTest, CancelLatencySample) { sample_aggregate_percentile: value: 50 concurrency_limit_params: - max_gradient: 2.0 max_concurrency_limit: concurrency_update_interval: 0.1s min_rtt_calc_params: @@ -286,7 +282,6 @@ TEST_F(GradientControllerTest, SamplePercentileProcessTest) { sample_aggregate_percentile: value: 50 concurrency_limit_params: - max_gradient: 2.0 max_concurrency_limit: concurrency_update_interval: 0.1s min_rtt_calc_params: @@ -307,12 +302,11 @@ TEST_F(GradientControllerTest, SamplePercentileProcessTest) { tryForward(controller, false); } -TEST_F(GradientControllerTest, ConcurrencyLimitBehaviorTestBasic) { +TEST_F(GradientControllerTest, MinRTTBufferTest) { const std::string yaml = R"EOF( sample_aggregate_percentile: value: 50 concurrency_limit_params: - max_gradient: 2.0 max_concurrency_limit: concurrency_update_interval: 0.1s min_rtt_calc_params: @@ -320,6 +314,8 @@ TEST_F(GradientControllerTest, ConcurrencyLimitBehaviorTestBasic) { value: 0.0 interval: 30s request_count: 5 + buffer: + value: 50 )EOF"; auto controller = makeController(yaml); @@ -330,43 +326,26 @@ TEST_F(GradientControllerTest, ConcurrencyLimitBehaviorTestBasic) { EXPECT_EQ( 5, stats_.gauge("test_prefix.min_rtt_msecs", Stats::Gauge::ImportMode::NeverImport).value()); - // Ensure that the concurrency window increases on its own due to the headroom calculation. - time_system_.sleep(std::chrono::milliseconds(101)); - dispatcher_->run(Event::Dispatcher::RunType::Block); - EXPECT_GT(controller->concurrencyLimit(), 3); - - // Make it seem as if the recorded latencies are consistently lower than the measured minRTT. - // Ensure that it grows. - for (int recalcs = 0; recalcs < 10; ++recalcs) { - const auto last_concurrency = controller->concurrencyLimit(); - for (int i = 1; i <= 5; ++i) { - tryForward(controller, true); - controller->recordLatencySample(std::chrono::milliseconds(4)); - } - time_system_.sleep(std::chrono::milliseconds(101)); - dispatcher_->run(Event::Dispatcher::RunType::Block); - EXPECT_GT(controller->concurrencyLimit(), last_concurrency); - } - - // Verify that the concurrency limit can now shrink as necessary. + // Ensure that the minRTT doesn't decrease due to the buffer added. for (int recalcs = 0; recalcs < 10; ++recalcs) { const auto last_concurrency = controller->concurrencyLimit(); for (int i = 1; i <= 5; ++i) { tryForward(controller, true); + // Recording sample that's technically higher than the minRTT, but the 50% buffer should + // prevent the concurrency limit from decreasing. controller->recordLatencySample(std::chrono::milliseconds(6)); } time_system_.sleep(std::chrono::milliseconds(101)); dispatcher_->run(Event::Dispatcher::RunType::Block); - EXPECT_LT(controller->concurrencyLimit(), last_concurrency); + EXPECT_GT(controller->concurrencyLimit(), last_concurrency); } } -TEST_F(GradientControllerTest, MaxGradientTest) { +TEST_F(GradientControllerTest, ConcurrencyLimitBehaviorTestBasic) { const std::string yaml = R"EOF( sample_aggregate_percentile: value: 50 concurrency_limit_params: - max_gradient: 3.0 max_concurrency_limit: concurrency_update_interval: 0.1s min_rtt_calc_params: @@ -374,28 +353,51 @@ TEST_F(GradientControllerTest, MaxGradientTest) { value: 0.0 interval: 30s request_count: 5 + buffer: + value: 10 )EOF"; auto controller = makeController(yaml); EXPECT_EQ(controller->concurrencyLimit(), 3); - // Force a minRTT of 5 seconds. - advancePastMinRTTStage(controller, yaml, std::chrono::seconds(5)); - - // circllhist approximates the percentiles, so we can expect it to be within a certain range. - EXPECT_THAT( - stats_.gauge("test_prefix.min_rtt_msecs", Stats::Gauge::ImportMode::NeverImport).value(), - AllOf(Ge(4950), Le(5050))); + // Force a minRTT of 5ms. + advancePastMinRTTStage(controller, yaml, std::chrono::milliseconds(5)); + EXPECT_EQ( + 5, stats_.gauge("test_prefix.min_rtt_msecs", Stats::Gauge::ImportMode::NeverImport).value()); - // Now verify max gradient value by forcing dramatically faster latency measurements.. - for (int i = 1; i <= 5; ++i) { - tryForward(controller, true); - controller->recordLatencySample(std::chrono::milliseconds(4)); - } + // Ensure that the concurrency window increases on its own due to the headroom calculation with + // the max gradient. time_system_.sleep(std::chrono::milliseconds(101)); dispatcher_->run(Event::Dispatcher::RunType::Block); - EXPECT_EQ(3.0, - stats_.gauge("test_prefix.gradient", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_GE(controller->concurrencyLimit(), 3); + EXPECT_LE(controller->concurrencyLimit() / 3.0, 2.0); + + // Make it seem as if the recorded latencies are consistently lower than the measured minRTT. + // Ensure that it grows. + for (int recalcs = 0; recalcs < 10; ++recalcs) { + const auto last_concurrency = controller->concurrencyLimit(); + for (int i = 1; i <= 5; ++i) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(4)); + } + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + // Verify the minimum gradient. + EXPECT_LE(last_concurrency, controller->concurrencyLimit()); + EXPECT_GE(static_cast(last_concurrency) / controller->concurrencyLimit(), 0.5); + } + + // Verify that the concurrency limit can now shrink as necessary. + for (int recalcs = 0; recalcs < 10; ++recalcs) { + const auto last_concurrency = controller->concurrencyLimit(); + for (int i = 1; i <= 5; ++i) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(6)); + } + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + EXPECT_LT(controller->concurrencyLimit(), last_concurrency); + } } TEST_F(GradientControllerTest, MinRTTReturnToPreviousLimit) { @@ -403,7 +405,6 @@ TEST_F(GradientControllerTest, MinRTTReturnToPreviousLimit) { sample_aggregate_percentile: value: 50 concurrency_limit_params: - max_gradient: 3.0 max_concurrency_limit: concurrency_update_interval: 0.1s min_rtt_calc_params: @@ -455,7 +456,6 @@ TEST_F(GradientControllerTest, MinRTTRescheduleTest) { sample_aggregate_percentile: value: 50 concurrency_limit_params: - max_gradient: 3.0 max_concurrency_limit: concurrency_update_interval: 0.1s min_rtt_calc_params: @@ -500,7 +500,6 @@ TEST_F(GradientControllerTest, NoSamplesTest) { sample_aggregate_percentile: value: 50 concurrency_limit_params: - max_gradient: 3.0 max_concurrency_limit: concurrency_update_interval: 0.1s min_rtt_calc_params: @@ -543,7 +542,6 @@ TEST_F(GradientControllerTest, TimerAccuracyTest) { sample_aggregate_percentile: value: 50 concurrency_limit_params: - max_gradient: 3.0 max_concurrency_limit: concurrency_update_interval: 0.123s min_rtt_calc_params: @@ -586,7 +584,6 @@ TEST_F(GradientControllerTest, TimerAccuracyTestNoJitter) { sample_aggregate_percentile: value: 50 concurrency_limit_params: - max_gradient: 3.0 max_concurrency_limit: concurrency_update_interval: 0.123s min_rtt_calc_params: From 6b602e50ec60b64bd42a35e2d7dad2c17c658627 Mon Sep 17 00:00:00 2001 From: Ismo Puustinen Date: Mon, 25 Nov 2019 19:56:50 +0200 Subject: [PATCH 27/29] tls: add function category() to private key factory header. (#9073) Suggested-by: James Peach Signed-off-by: Ismo Puustinen --- include/envoy/config/typed_metadata.h | 2 ++ .../ssl/private_key/private_key_config.h | 19 +++++++++++++++++++ test/test_common/registry.h | 1 + 3 files changed, 22 insertions(+) diff --git a/include/envoy/config/typed_metadata.h b/include/envoy/config/typed_metadata.h index 47abbcdeeebc..369ec0927bd3 100644 --- a/include/envoy/config/typed_metadata.h +++ b/include/envoy/config/typed_metadata.h @@ -72,6 +72,8 @@ class TypedMetadataFactory { */ virtual std::unique_ptr parse(const ProtobufWkt::Struct& data) const PURE; + + static std::string category() { return "typed_metadata"; } }; } // namespace Config diff --git a/include/envoy/ssl/private_key/private_key_config.h b/include/envoy/ssl/private_key/private_key_config.h index 8a5da737cac4..6f6eef9ac0e2 100644 --- a/include/envoy/ssl/private_key/private_key_config.h +++ b/include/envoy/ssl/private_key/private_key_config.h @@ -12,10 +12,29 @@ namespace Ssl { class PrivateKeyMethodProviderInstanceFactory { public: virtual ~PrivateKeyMethodProviderInstanceFactory() = default; + + /** + * Create a particular PrivateKeyMethodProvider implementation. If the implementation is + * unable to produce a PrivateKeyMethodProvider with the provided parameters, it should throw + * an EnvoyException. The returned pointer should always be valid. + * @param config supplies the custom proto configuration for the PrivateKeyMethodProvider + * @param context supplies the factory context + */ virtual PrivateKeyMethodProviderSharedPtr createPrivateKeyMethodProviderInstance( const envoy::api::v2::auth::PrivateKeyProvider& config, Server::Configuration::TransportSocketFactoryContext& factory_context) PURE; + + /** + * @return std::string the identifying name for a particular implementation of + * PrivateKeyMethodProvider produced by the factory. + */ virtual std::string name() const PURE; + + /** + * @return std::string the identifying category name for objects created by this factory. + * Used for automatic registration with FactoryCategoryRegistry. + */ + static std::string category() { return "tls.key_providers"; }; }; } // namespace Ssl diff --git a/test/test_common/registry.h b/test/test_common/registry.h index 0cdcd8f5afd7..6c58095594dc 100644 --- a/test/test_common/registry.h +++ b/test/test_common/registry.h @@ -15,6 +15,7 @@ namespace Registry { template class InjectFactory { public: InjectFactory(Base& instance) : instance_(instance) { + EXPECT_STRNE(Base::category().c_str(), ""); displaced_ = Registry::FactoryRegistry::replaceFactoryForTest(instance_); } From 95dfc516d987aecdd29b381f957c8186c3c93436 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 25 Nov 2019 09:59:09 -0800 Subject: [PATCH 28/29] init: add shared target (#9008) Extracted from #8523 @mergeconflict mentioned we can also maintain a collection of TargetImpl(and a initialize_ flag, std::once_flag) at the target owner. I agree it works but a SharedTarget provides stronger cohesion. Follow up: * update #8523 on top of this PR * complete fuzz test #8714 Signed-off-by: Yuchen Dai --- source/common/init/target_impl.cc | 34 +++++++- source/common/init/target_impl.h | 46 ++++++++++ test/common/init/target_impl_test.cc | 126 +++++++++++++++++++++++++-- test/mocks/init/mocks.cc | 7 ++ test/mocks/init/mocks.h | 12 +++ 5 files changed, 215 insertions(+), 10 deletions(-) diff --git a/source/common/init/target_impl.cc b/source/common/init/target_impl.cc index 5bf0288b8298..4d8df4c27aac 100644 --- a/source/common/init/target_impl.cc +++ b/source/common/init/target_impl.cc @@ -35,7 +35,7 @@ absl::string_view TargetImpl::name() const { return name_; } TargetHandlePtr TargetImpl::createHandle(absl::string_view handle_name) const { // Note: can't use std::make_unique here because TargetHandleImpl ctor is private. - return std::unique_ptr( + return TargetHandlePtr( new TargetHandleImpl(handle_name, name_, std::weak_ptr(fn_))); } @@ -50,5 +50,37 @@ bool TargetImpl::ready() { return false; } +SharedTargetImpl::SharedTargetImpl(absl::string_view name, InitializeFn fn) + : name_(fmt::format("shared target {}", name)), + fn_(std::make_shared([this, fn](WatcherHandlePtr watcher_handle) { + if (initialized_) { + watcher_handle->ready(); + } else { + watcher_handles_.push_back(std::move(watcher_handle)); + std::call_once(once_flag_, fn); + } + })) {} + +SharedTargetImpl::~SharedTargetImpl() { ENVOY_LOG(debug, "{} destroyed", name_); } + +absl::string_view SharedTargetImpl::name() const { return name_; } + +TargetHandlePtr SharedTargetImpl::createHandle(absl::string_view handle_name) const { + // Note: can't use std::make_unique here because TargetHandleImpl ctor is private. + return TargetHandlePtr( + new TargetHandleImpl(handle_name, name_, std::weak_ptr(fn_))); +} + +bool SharedTargetImpl::ready() { + initialized_ = true; + bool all_notified = !watcher_handles_.empty(); + for (auto& watcher_handle : watcher_handles_) { + all_notified = watcher_handle->ready() && all_notified; + } + // save heap and avoid repeatedly invoke + watcher_handles_.clear(); + return all_notified; +} + } // namespace Init } // namespace Envoy diff --git a/source/common/init/target_impl.h b/source/common/init/target_impl.h index ad2757433f0d..d6a098daaca2 100644 --- a/source/common/init/target_impl.h +++ b/source/common/init/target_impl.h @@ -29,6 +29,8 @@ using InternalInitalizeFn = std::function; class TargetHandleImpl : public TargetHandle, Logger::Loggable { private: friend class TargetImpl; + friend class SharedTargetImpl; + TargetHandleImpl(absl::string_view handle_name, absl::string_view name, std::weak_ptr fn); @@ -85,5 +87,49 @@ class TargetImpl : public Target, Logger::Loggable { const std::shared_ptr fn_; }; +/** + * A specialized Target which can be added by multiple Managers. + * The initialization will be triggered only once. + */ +class SharedTargetImpl : public Target, Logger::Loggable { +public: + /** + * @param name a human-readable target name, for logging / debugging + * @fn a callback function to invoke when `initialize` is called on the handle. Note that this + * doesn't take a WatcherHandlePtr (like TargetFn does). Managing the watcher handle is done + * internally to simplify usage. + */ + SharedTargetImpl(absl::string_view name, InitializeFn fn); + ~SharedTargetImpl() override; + + // Init::Target + absl::string_view name() const override; + TargetHandlePtr createHandle(absl::string_view handle_name) const override; + + /** + * Signal to the init manager(s) that this target has finished initializing. This is safe to call + * any time. Calling it before initialization begins or after initialization has already ended + * will have no effect. + * @return true if all init managers received this call, false otherwise. + */ + bool ready(); + +private: + // Human-readable name for logging + const std::string name_; + + // Handle to all the ManagerImpl's internal watcher, to call when this target is initialized. + std::vector watcher_handles_; + + // The callback function, called via TargetHandleImpl by the manager + const std::shared_ptr fn_; + + // The state so as to signal the manager when a ready target is added. + bool initialized_{false}; + + // To guarantee the initialization function is called once. + std::once_flag once_flag_; +}; + } // namespace Init } // namespace Envoy diff --git a/test/common/init/target_impl_test.cc b/test/common/init/target_impl_test.cc index 7cebbb371d9e..820556deea0d 100644 --- a/test/common/init/target_impl_test.cc +++ b/test/common/init/target_impl_test.cc @@ -8,15 +8,22 @@ namespace Envoy { namespace Init { namespace { -TEST(InitTargetImplTest, Name) { - ExpectableTargetImpl target; - EXPECT_EQ("target test", target.name()); +// Testing common cases for all the target implementation. +template class TargetImplTest : public ::testing::Test {}; +TYPED_TEST_SUITE_P(TargetImplTest); + +template std::string getName() { return ""; } +template <> std::string getName() { return "target test"; } +template <> std::string getName() { return "shared target test"; } +TYPED_TEST_P(TargetImplTest, Name) { + TypeParam target; + EXPECT_EQ(getName(), target.name()); } -TEST(InitTargetImplTest, InitializeWhenAvailable) { +TYPED_TEST_P(TargetImplTest, InitializeWhenAvailable) { InSequence s; - ExpectableTargetImpl target; + TypeParam target; ExpectableWatcherImpl watcher; // initializing the target through its handle should invoke initialize()... @@ -32,21 +39,24 @@ TEST(InitTargetImplTest, InitializeWhenAvailable) { EXPECT_FALSE(target.ready()); } -TEST(InitTargetImplTest, InitializeWhenUnavailable) { +// Initializing TargetHandle return false if uninitialized SharedTarget is destroyed. +TYPED_TEST_P(TargetImplTest, InitializeWhenUnavailable) { + InSequence s; ExpectableWatcherImpl watcher; TargetHandlePtr handle; { - ExpectableTargetImpl target; + TypeParam target; // initializing the target after it's been destroyed should do nothing. handle = target.createHandle("test"); target.expectInitialize().Times(0); + // target destroyed here } EXPECT_FALSE(handle->initialize(watcher)); } -TEST(InitTargetImplTest, ReadyWhenWatcherUnavailable) { - ExpectableTargetImpl target; +TYPED_TEST_P(TargetImplTest, ReadyWhenWatcherUnavailable) { + TypeParam target; { ExpectableWatcherImpl watcher; @@ -56,10 +66,108 @@ TEST(InitTargetImplTest, ReadyWhenWatcherUnavailable) { // calling ready() on the target after the watcher has been destroyed should do nothing. watcher.expectReady().Times(0); + // watcher destroyed here } EXPECT_FALSE(target.ready()); } +REGISTER_TYPED_TEST_SUITE_P(TargetImplTest, Name, InitializeWhenAvailable, + InitializeWhenUnavailable, ReadyWhenWatcherUnavailable); +using TargetImplTypes = ::testing::Types; +INSTANTIATE_TYPED_TEST_SUITE_P(Init, TargetImplTest, TargetImplTypes); + +TYPED_TEST_SUITE(TargetImplTest, TargetImplTypes); + +// Below are the specialized tests for different implementations of Target + +// Initializing TargetHandle return false if initialized SharedTarget is destroyed. +TEST(SharedTargetImplTest, ReInitializeWhenUnavailable) { + InSequence s; + ExpectableWatcherImpl w; + TargetHandlePtr handle; + { + ExpectableSharedTargetImpl target; + + target.expectInitialize(); + TargetHandlePtr handle1 = target.createHandle("m1"); + ExpectableWatcherImpl w1; + EXPECT_TRUE(handle1->initialize(w1)); + + // initializing the target after it's been destroyed should do nothing. + handle = target.createHandle("m2"); + target.expectInitialize().Times(0); + // target destroyed + } + EXPECT_FALSE(handle->initialize(w)); +} + +// SharedTarget notifies multiple watchers. +TEST(SharedTargetImplTest, NotifyAllWatcherWhenInitialization) { + InSequence s; + ExpectableWatcherImpl w1; + ExpectableSharedTargetImpl target; + + target.expectInitialize(); + TargetHandlePtr handle1 = target.createHandle("m1"); + EXPECT_TRUE(handle1->initialize(w1)); + + ExpectableWatcherImpl w2; + target.expectInitialize().Times(0); + TargetHandlePtr handle2 = target.createHandle("m2"); + // calling ready() on the target should invoke all the saved watchers. + w1.expectReady(); + EXPECT_TRUE(target.ready()); + w2.expectReady(); + EXPECT_TRUE(handle2->initialize(w2)); +} + +// Initialized SharedTarget notifies further watcher immediately at second initialization attempt. +TEST(SharedTargetImplTest, InitializedSharedTargetNotifyWatcherWhenAddedAgain) { + InSequence s; + ExpectableWatcherImpl w1; + ExpectableSharedTargetImpl target; + + target.expectInitialize(); + TargetHandlePtr handle1 = target.createHandle("m1"); + EXPECT_TRUE(handle1->initialize(w1)); + + // calling ready() on the target should invoke the saved watcher handle(s). + w1.expectReady(); + EXPECT_TRUE(target.ready()); + + ExpectableWatcherImpl w2; + target.expectInitialize().Times(0); + TargetHandlePtr handle2 = target.createHandle("m2"); + // w2 is notified with no further target.ready(). + w2.expectReady(); + EXPECT_TRUE(handle2->initialize(w2)); +} + +TEST(SharedTargetImplTest, EarlySharedTargetReadyNotifyWatchers) { + InSequence s; + + ExpectableSharedTargetImpl target; + + // No watcher yet. Nothing will be notified at this moment. + EXPECT_FALSE(target.ready()); + + // It's arguable if the shared target should be initialized after ready() + // is already invoked. + target.expectInitialize().Times(0); + + ExpectableWatcherImpl w1; + TargetHandlePtr handle1 = target.createHandle("m1"); + // w1 is notified with no further target.ready(). + w1.expectReady(); + EXPECT_TRUE(handle1->initialize(w1)); + + ExpectableWatcherImpl w2; + target.expectInitialize().Times(0); + TargetHandlePtr handle2 = target.createHandle("m2"); + // w2 is notified with no further target.ready(). + w2.expectReady(); + EXPECT_TRUE(handle2->initialize(w2)); +} } // namespace } // namespace Init } // namespace Envoy diff --git a/test/mocks/init/mocks.cc b/test/mocks/init/mocks.cc index 9e28923af3c0..756611640ff0 100644 --- a/test/mocks/init/mocks.cc +++ b/test/mocks/init/mocks.cc @@ -21,5 +21,12 @@ ExpectableTargetImpl::expectInitializeWillCallReady() { return expectInitialize().WillOnce(Invoke([this]() { ready(); })); } +ExpectableSharedTargetImpl::ExpectableSharedTargetImpl(absl::string_view name) + : ExpectableSharedTargetImpl(name, [this]() { initialize(); }) {} +ExpectableSharedTargetImpl::ExpectableSharedTargetImpl(absl::string_view name, InitializeFn fn) + : SharedTargetImpl(name, fn) {} +::testing::internal::TypedExpectation& ExpectableSharedTargetImpl::expectInitialize() { + return EXPECT_CALL(*this, initialize()); +} } // namespace Init } // namespace Envoy diff --git a/test/mocks/init/mocks.h b/test/mocks/init/mocks.h index 44189ac09144..5c49443156a0 100644 --- a/test/mocks/init/mocks.h +++ b/test/mocks/init/mocks.h @@ -50,6 +50,18 @@ class ExpectableTargetImpl : public TargetImpl { ::testing::internal::TypedExpectation& expectInitializeWillCallReady(); }; +/** + * Borrow the idea from ExpectableTargetImpl. ExpectableSharedTargetImpl is a real SharedTargetImpl. + */ +class ExpectableSharedTargetImpl : public SharedTargetImpl { +public: + ExpectableSharedTargetImpl(absl::string_view name = "test"); + ExpectableSharedTargetImpl(absl::string_view name, InitializeFn fn); + MOCK_METHOD0(initialize, void()); + + ::testing::internal::TypedExpectation& expectInitialize(); +}; + /** * MockManager is a typical mock. In many cases, it won't be necessary to mock any of its methods. * In cases where its `add` and `initialize` methods are actually called in a test, it's usually From 647c1eeba8622bafdd6add1e7997c1f0bda31be5 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Mon, 25 Nov 2019 10:25:25 -0800 Subject: [PATCH 29/29] udp_proxy: implement idle timeout and some stats (#8999) Another bunch of work towards https://github.com/envoyproxy/envoy/issues/492. The remaining work is proper wiring up of upstream cluster management, host health, etc. and documentation. This will be done in the next PR. Signed-off-by: Matt Klein --- .../udp/udp_proxy/v2alpha/udp_proxy.proto | 12 +- include/envoy/network/filter.h | 7 + include/envoy/network/listener.h | 7 +- source/common/network/udp_listener_impl.cc | 6 +- source/common/network/utility.cc | 2 +- .../extensions/filters/udp/udp_proxy/config.h | 2 +- .../filters/udp/udp_proxy/udp_proxy_filter.cc | 67 +++-- .../filters/udp/udp_proxy/udp_proxy_filter.h | 56 +++- .../quiche/active_quic_listener.h | 3 +- source/server/connection_handler_impl.cc | 7 +- source/server/connection_handler_impl.h | 3 +- test/common/network/udp_listener_impl_test.cc | 33 +-- test/extensions/filters/udp/udp_proxy/BUILD | 12 + .../udp/udp_proxy/udp_proxy_filter_test.cc | 248 ++++++++++++++++++ .../udp_proxy/udp_proxy_integration_test.cc | 13 +- test/integration/fake_upstream.h | 1 + test/mocks/network/BUILD | 9 + test/mocks/network/io_handle.cc | 12 + test/mocks/network/io_handle.h | 29 ++ test/mocks/network/mocks.cc | 5 +- test/mocks/network/mocks.h | 18 +- test/test_common/network_utility.cc | 2 +- test/test_common/network_utility.h | 3 +- 23 files changed, 480 insertions(+), 77 deletions(-) create mode 100644 test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc create mode 100644 test/mocks/network/io_handle.cc create mode 100644 test/mocks/network/io_handle.h diff --git a/api/envoy/config/filter/udp/udp_proxy/v2alpha/udp_proxy.proto b/api/envoy/config/filter/udp/udp_proxy/v2alpha/udp_proxy.proto index 5f2db4a91814..68603f38d327 100644 --- a/api/envoy/config/filter/udp/udp_proxy/v2alpha/udp_proxy.proto +++ b/api/envoy/config/filter/udp/udp_proxy/v2alpha/udp_proxy.proto @@ -12,11 +12,19 @@ import "validate/validate.proto"; // TODO(mattklein123): docs +// Configuration for the UDP proxy filter. message UdpProxyConfig { - oneof cluster_specifier { + // The stat prefix used when emitting UDP proxy filter stats. + string stat_prefix = 1 [(validate.rules).string = {min_bytes: 1}]; + + oneof route_specifier { option (validate.required) = true; // The upstream cluster to connect to. - string cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + string cluster = 2 [(validate.rules).string = {min_bytes: 1}]; } + + // The idle timeout for sessions. Idle is defined as no datagrams between received or sent by + // the session. The default if not specified is 1 minute. + google.protobuf.Duration idle_timeout = 3; } diff --git a/include/envoy/network/filter.h b/include/envoy/network/filter.h index 0019ccedcec3..42cd0a945779 100644 --- a/include/envoy/network/filter.h +++ b/include/envoy/network/filter.h @@ -378,6 +378,13 @@ class UdpListenerReadFilter { */ virtual void onData(UdpRecvData& data) PURE; + /** + * Called when there is an error event in the receive data path. + * + * @param error_code supplies the received error on the listener. + */ + virtual void onReceiveError(Api::IoError::IoErrorCode error_code) PURE; + protected: /** * @param callbacks supplies the read filter callbacks used to interact with the filter manager. diff --git a/include/envoy/network/listener.h b/include/envoy/network/listener.h index 44067f5ce202..162c79aae80f 100644 --- a/include/envoy/network/listener.h +++ b/include/envoy/network/listener.h @@ -201,8 +201,6 @@ struct UdpSendData { */ class UdpListenerCallbacks { public: - enum class ErrorCode { SyscallError, UnknownError }; - virtual ~UdpListenerCallbacks() = default; /** @@ -225,10 +223,9 @@ class UdpListenerCallbacks { * Called when there is an error event in the receive data path. * The send side error is a return type on the send method. * - * @param error_code ErrorCode for the error event. - * @param error_number System error number. + * @param error_code supplies the received error on the listener. */ - virtual void onReceiveError(const ErrorCode& error_code, Api::IoError::IoErrorCode err) PURE; + virtual void onReceiveError(Api::IoError::IoErrorCode error_code) PURE; }; /** diff --git a/source/common/network/udp_listener_impl.cc b/source/common/network/udp_listener_impl.cc index 66fd27ef978d..d864c85032a7 100644 --- a/source/common/network/udp_listener_impl.cc +++ b/source/common/network/udp_listener_impl.cc @@ -72,9 +72,11 @@ void UdpListenerImpl::handleReadCallback() { socket_->ioHandle(), *socket_->localAddress(), *this, time_source_, packets_dropped_); // TODO(mattklein123): Handle no error when we limit the number of packets read. if (result->getErrorCode() != Api::IoError::IoErrorCode::Again) { - ENVOY_UDP_LOG(error, "recvmsg result {}: {}", static_cast(result->getErrorCode()), + // TODO(mattklein123): When rate limited logging is implemented log this at error level + // on a periodic basis. + ENVOY_UDP_LOG(debug, "recvmsg result {}: {}", static_cast(result->getErrorCode()), result->getErrorDetails()); - cb_.onReceiveError(UdpListenerCallbacks::ErrorCode::SyscallError, result->getErrorCode()); + cb_.onReceiveError(result->getErrorCode()); } } diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index 4392de6e9792..b288bf41a93d 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -514,7 +514,7 @@ Api::IoCallUint64Result Utility::writeToSocket(IoHandle& handle, Buffer::RawSlic send_result.err_->getErrorCode() == Api::IoError::IoErrorCode::Interrupt); if (send_result.ok()) { - ENVOY_LOG_MISC(trace, "sendmsg sent:{} bytes", send_result.rc_); + ENVOY_LOG_MISC(trace, "sendmsg bytes {}", send_result.rc_); } else { ENVOY_LOG_MISC(debug, "sendmsg failed with error code {}: {}", static_cast(send_result.err_->getErrorCode()), diff --git a/source/extensions/filters/udp/udp_proxy/config.h b/source/extensions/filters/udp/udp_proxy/config.h index df69432eef07..2e6178c99b11 100644 --- a/source/extensions/filters/udp/udp_proxy/config.h +++ b/source/extensions/filters/udp/udp_proxy/config.h @@ -22,7 +22,7 @@ class UdpProxyFilterConfigFactory createFilterFactoryFromProto(const Protobuf::Message& config, Server::Configuration::ListenerFactoryContext& context) override { auto shared_config = std::make_shared( - context.clusterManager(), context.timeSource(), + context.clusterManager(), context.timeSource(), context.scope(), MessageUtil::downcastAndValidate< const envoy::config::filter::udp::udp_proxy::v2alpha::UdpProxyConfig&>( config, context.messageValidationVisitor())); diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc index 315277b7253d..ad2eef579090 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc @@ -7,9 +7,6 @@ namespace Extensions { namespace UdpFilters { namespace UdpProxy { -// TODO(mattklein123): Logging -// TODO(mattklein123): Stats - void UdpProxyFilter::onData(Network::UdpRecvData& data) { const auto active_session_it = sessions_.find(data.addresses_); ActiveSession* active_session; @@ -18,7 +15,7 @@ void UdpProxyFilter::onData(Network::UdpRecvData& data) { // TODO(mattklein123): Instead of looking up the cluster each time, keep track of it via // cluster manager callbacks. Upstream::ThreadLocalCluster* cluster = config_->getCluster(); - // TODO(mattklein123): Handle the case where the cluster does not exist. + // TODO(mattklein123): Handle the case where the cluster does not exist and add stat. ASSERT(cluster != nullptr); // TODO(mattklein123): Pass a context and support hash based routing. @@ -37,18 +34,26 @@ void UdpProxyFilter::onData(Network::UdpRecvData& data) { active_session->write(*data.buffer_); } +void UdpProxyFilter::onReceiveError(Api::IoError::IoErrorCode) { + config_->stats().downstream_sess_rx_errors_.inc(); +} + UdpProxyFilter::ActiveSession::ActiveSession(UdpProxyFilter& parent, Network::UdpRecvData::LocalPeerAddresses&& addresses, const Upstream::HostConstSharedPtr& host) : parent_(parent), addresses_(std::move(addresses)), host_(host), + idle_timer_(parent.read_callbacks_->udpListener().dispatcher().createTimer( + [this] { onIdleTimer(); })), // NOTE: The socket call can only fail due to memory/fd exhaustion. No local ephemeral port // is bound until the first packet is sent to the upstream host. - io_handle_(host->address()->socket(Network::Address::SocketType::Datagram)), + io_handle_(parent.createIoHandle(host)), socket_event_(parent.read_callbacks_->udpListener().dispatcher().createFileEvent( io_handle_->fd(), [this](uint32_t) { onReadReady(); }, Event::FileTriggerType::Edge, Event::FileReadyType::Read)) { ENVOY_LOG(debug, "creating new session: downstream={} local={}", addresses_.peer_->asStringView(), addresses_.local_->asStringView()); + parent_.config_->stats().downstream_sess_total_.inc(); + parent_.config_->stats().downstream_sess_active_.inc(); // TODO(mattklein123): Enable dropped packets socket option. In general the Socket abstraction // does not work well right now for client sockets. It's too heavy weight and is aimed at listener @@ -57,39 +62,71 @@ UdpProxyFilter::ActiveSession::ActiveSession(UdpProxyFilter& parent, // handle. } +UdpProxyFilter::ActiveSession::~ActiveSession() { + parent_.config_->stats().downstream_sess_active_.dec(); +} + +void UdpProxyFilter::ActiveSession::onIdleTimer() { + ENVOY_LOG(debug, "session idle timeout: downstream={} local={}", addresses_.peer_->asStringView(), + addresses_.local_->asStringView()); + parent_.config_->stats().idle_timeout_.inc(); + parent_.sessions_.erase(addresses_); +} + void UdpProxyFilter::ActiveSession::onReadReady() { - // TODO(mattklein123): Refresh idle timer. + idle_timer_->enableTimer(parent_.config_->sessionTimeout()); + + // TODO(mattklein123): We should not be passing *addresses_.local_ to this function as we are + // not trying to populate the local address for received packets. uint32_t packets_dropped = 0; const Api::IoErrorPtr result = Network::Utility::readPacketsFromSocket( *io_handle_, *addresses_.local_, *this, parent_.config_->timeSource(), packets_dropped); // TODO(mattklein123): Handle no error when we limit the number of packets read. - // TODO(mattklein123): Increment stat on failure. - ASSERT(result->getErrorCode() == Api::IoError::IoErrorCode::Again); + if (result->getErrorCode() != Api::IoError::IoErrorCode::Again) { + // TODO(mattklein123): Upstream cluster RX error stat. + } } void UdpProxyFilter::ActiveSession::write(const Buffer::Instance& buffer) { - ENVOY_LOG(trace, "writing {} byte datagram: downstream={} local={} upstream={}", buffer.length(), - addresses_.peer_->asStringView(), addresses_.local_->asStringView(), + ENVOY_LOG(trace, "writing {} byte datagram upstream: downstream={} local={} upstream={}", + buffer.length(), addresses_.peer_->asStringView(), addresses_.local_->asStringView(), host_->address()->asStringView()); + parent_.config_->stats().downstream_sess_rx_bytes_.add(buffer.length()); + parent_.config_->stats().downstream_sess_rx_datagrams_.inc(); + + idle_timer_->enableTimer(parent_.config_->sessionTimeout()); - // TODO(mattklein123): Refresh idle timer. // NOTE: On the first write, a local ephemeral port is bound, and thus this write can fail due to // port exhaustion. // NOTE: We do not specify the local IP to use for the sendmsg call. We allow the OS to select // the right IP based on outbound routing rules. Api::IoCallUint64Result rc = Network::Utility::writeToSocket(*io_handle_, buffer, nullptr, *host_->address()); - // TODO(mattklein123): Increment stat on failure. - ASSERT(rc.ok()); + if (!rc.ok()) { + // TODO(mattklein123): Upstream cluster TX error stat. + } else { + // TODO(mattklein123): Upstream cluster TX byte/datagram stats. + } } void UdpProxyFilter::ActiveSession::processPacket(Network::Address::InstanceConstSharedPtr, Network::Address::InstanceConstSharedPtr, Buffer::InstancePtr buffer, MonotonicTime) { + ENVOY_LOG(trace, "writing {} byte datagram downstream: downstream={} local={} upstream={}", + buffer->length(), addresses_.peer_->asStringView(), addresses_.local_->asStringView(), + host_->address()->asStringView()); + const uint64_t buffer_length = buffer->length(); + + // TODO(mattklein123): Upstream cluster RX byte/datagram stats. + Network::UdpSendData data{addresses_.local_->ip(), *addresses_.peer_, *buffer}; const Api::IoCallUint64Result rc = parent_.read_callbacks_->udpListener().send(data); - // TODO(mattklein123): Increment stat on failure. - ASSERT(rc.ok()); + if (!rc.ok()) { + parent_.config_->stats().downstream_sess_tx_errors_.inc(); + } else { + parent_.config_->stats().downstream_sess_tx_bytes_.add(buffer_length); + parent_.config_->stats().downstream_sess_tx_datagrams_.inc(); + } } } // namespace UdpProxy diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h index 9ee9188bb460..e248dafb75cc 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h @@ -15,21 +15,53 @@ namespace Extensions { namespace UdpFilters { namespace UdpProxy { +/** + * All UDP proxy stats. @see stats_macros.h + */ +#define ALL_UDP_PROXY_STATS(COUNTER, GAUGE) \ + COUNTER(downstream_sess_rx_bytes) \ + COUNTER(downstream_sess_rx_datagrams) \ + COUNTER(downstream_sess_rx_errors) \ + COUNTER(downstream_sess_total) \ + COUNTER(downstream_sess_tx_bytes) \ + COUNTER(downstream_sess_tx_datagrams) \ + COUNTER(downstream_sess_tx_errors) \ + COUNTER(idle_timeout) \ + GAUGE(downstream_sess_active, Accumulate) + +/** + * Struct definition for all UDP proxy stats. @see stats_macros.h + */ +struct UdpProxyStats { + ALL_UDP_PROXY_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) +}; + class UdpProxyFilterConfig { public: UdpProxyFilterConfig(Upstream::ClusterManager& cluster_manager, TimeSource& time_source, + Stats::Scope& root_scope, const envoy::config::filter::udp::udp_proxy::v2alpha::UdpProxyConfig& config) - : cluster_manager_(cluster_manager), time_source_(time_source), config_(config) {} + : cluster_manager_(cluster_manager), time_source_(time_source), cluster_(config.cluster()), + session_timeout_(PROTOBUF_GET_MS_OR_DEFAULT(config, idle_timeout, 60 * 1000)), + stats_(generateStats(config.stat_prefix(), root_scope)) {} - Upstream::ThreadLocalCluster* getCluster() const { - return cluster_manager_.get(config_.cluster()); - } + Upstream::ThreadLocalCluster* getCluster() const { return cluster_manager_.get(cluster_); } + std::chrono::milliseconds sessionTimeout() const { return session_timeout_; } + UdpProxyStats& stats() const { return stats_; } TimeSource& timeSource() const { return time_source_; } private: + static UdpProxyStats generateStats(const std::string& stat_prefix, Stats::Scope& scope) { + const auto final_prefix = fmt::format("udp.{}", stat_prefix); + return {ALL_UDP_PROXY_STATS(POOL_COUNTER_PREFIX(scope, final_prefix), + POOL_GAUGE_PREFIX(scope, final_prefix))}; + } + Upstream::ClusterManager& cluster_manager_; TimeSource& time_source_; - const envoy::config::filter::udp::udp_proxy::v2alpha::UdpProxyConfig config_; + const std::string cluster_; + const std::chrono::milliseconds session_timeout_; + mutable UdpProxyStats stats_; }; using UdpProxyFilterConfigSharedPtr = std::shared_ptr; @@ -42,6 +74,7 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, Logger::Loggableaddress()->socket(Network::Address::SocketType::Datagram); + } + const UdpProxyFilterConfigSharedPtr config_; absl::flat_hash_set diff --git a/source/extensions/quic_listeners/quiche/active_quic_listener.h b/source/extensions/quic_listeners/quiche/active_quic_listener.h index fc14a490e242..c4e52ae7d319 100644 --- a/source/extensions/quic_listeners/quiche/active_quic_listener.h +++ b/source/extensions/quic_listeners/quiche/active_quic_listener.h @@ -32,8 +32,7 @@ class ActiveQuicListener : public Network::UdpListenerCallbacks, // Network::UdpListenerCallbacks void onData(Network::UdpRecvData& data) override; void onWriteReady(const Network::Socket& socket) override; - void onReceiveError(const Network::UdpListenerCallbacks::ErrorCode& /*error_code*/, - Api::IoError::IoErrorCode /*err*/) override { + void onReceiveError(Api::IoError::IoErrorCode /*error_code*/) override { // No-op. Quic can't do anything upon listener error. } diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index d4c272f95dc4..d47340ee1640 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -438,11 +438,8 @@ void ActiveUdpListener::onWriteReady(const Network::Socket&) { // data } -void ActiveUdpListener::onReceiveError(const Network::UdpListenerCallbacks::ErrorCode&, - Api::IoError::IoErrorCode) { - // TODO(sumukhs): Determine what to do on receive error. - // Would the filters need to know on error? Can't foresee a scenario where they - // would take an action +void ActiveUdpListener::onReceiveError(Api::IoError::IoErrorCode error_code) { + read_filter_->onReceiveError(error_code); } void ActiveUdpListener::addReadFilter(Network::UdpListenerReadFilterPtr&& filter) { diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index cc809fcf4ea3..35c52a849c1d 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -262,8 +262,7 @@ class ActiveUdpListener : public Network::UdpListenerCallbacks, // Network::UdpListenerCallbacks void onData(Network::UdpRecvData& data) override; void onWriteReady(const Network::Socket& socket) override; - void onReceiveError(const Network::UdpListenerCallbacks::ErrorCode& error_code, - Api::IoError::IoErrorCode err) override; + void onReceiveError(Api::IoError::IoErrorCode error_code) override; // ActiveListenerImplBase Network::Listener* listener() override { return udp_listener_.get(); } diff --git a/test/common/network/udp_listener_impl_test.cc b/test/common/network/udp_listener_impl_test.cc index 94408da5242e..485df4fef59d 100644 --- a/test/common/network/udp_listener_impl_test.cc +++ b/test/common/network/udp_listener_impl_test.cc @@ -146,7 +146,7 @@ TEST_P(UdpListenerImplTest, UseActualDstUdp) { *send_to_addr_); ASSERT_EQ(send_rc.rc_, second.length()); - EXPECT_CALL(listener_callbacks_, onData_(_)) + EXPECT_CALL(listener_callbacks_, onData(_)) .WillOnce(Invoke([&](const UdpRecvData& data) -> void { validateRecvCallbackParams(data); @@ -160,7 +160,7 @@ TEST_P(UdpListenerImplTest, UseActualDstUdp) { dispatcher_->exit(); })); - EXPECT_CALL(listener_callbacks_, onWriteReady_(_)) + EXPECT_CALL(listener_callbacks_, onWriteReady(_)) .WillRepeatedly(Invoke([&](const Socket& socket) { EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); })); @@ -196,7 +196,7 @@ TEST_P(UdpListenerImplTest, UdpEcho) { std::vector server_received_data; - EXPECT_CALL(listener_callbacks_, onData_(_)) + EXPECT_CALL(listener_callbacks_, onData(_)) .WillOnce(Invoke([&](const UdpRecvData& data) -> void { validateRecvCallbackParams(data); @@ -216,7 +216,7 @@ TEST_P(UdpListenerImplTest, UdpEcho) { server_received_data.push_back(data_str); })); - EXPECT_CALL(listener_callbacks_, onWriteReady_(_)).WillOnce(Invoke([&](const Socket& socket) { + EXPECT_CALL(listener_callbacks_, onWriteReady(_)).WillOnce(Invoke([&](const Socket& socket) { EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); ASSERT_NE(test_peer_address, nullptr); @@ -283,15 +283,15 @@ TEST_P(UdpListenerImplTest, UdpListenerEnableDisable) { *send_to_addr_); ASSERT_EQ(send_rc.rc_, second.length()); - EXPECT_CALL(listener_callbacks_, onData_(_)).Times(0); + EXPECT_CALL(listener_callbacks_, onData(_)).Times(0); - EXPECT_CALL(listener_callbacks_, onWriteReady_(_)).Times(0); + EXPECT_CALL(listener_callbacks_, onWriteReady(_)).Times(0); dispatcher_->run(Event::Dispatcher::RunType::Block); listener_->enable(); - EXPECT_CALL(listener_callbacks_, onData_(_)) + EXPECT_CALL(listener_callbacks_, onData(_)) .Times(2) .WillOnce(Return()) .WillOnce(Invoke([&](const UdpRecvData& data) -> void { @@ -302,7 +302,7 @@ TEST_P(UdpListenerImplTest, UdpListenerEnableDisable) { dispatcher_->exit(); })); - EXPECT_CALL(listener_callbacks_, onWriteReady_(_)) + EXPECT_CALL(listener_callbacks_, onWriteReady(_)) .WillRepeatedly(Invoke([&](const Socket& socket) { EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); })); @@ -329,19 +329,14 @@ TEST_P(UdpListenerImplTest, UdpListenerRecvMsgError) { nullptr, *send_to_addr_); ASSERT_EQ(send_rc.rc_, first.length()); - EXPECT_CALL(listener_callbacks_, onData_(_)).Times(0); + EXPECT_CALL(listener_callbacks_, onData(_)).Times(0); - EXPECT_CALL(listener_callbacks_, onWriteReady_(_)) - .Times(1) - .WillRepeatedly(Invoke([&](const Socket& socket) { - EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); - })); + EXPECT_CALL(listener_callbacks_, onWriteReady(_)).WillOnce(Invoke([&](const Socket& socket) { + EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); + })); - EXPECT_CALL(listener_callbacks_, onReceiveError_(_, _)) - .Times(1) - .WillOnce(Invoke([&](const UdpListenerCallbacks::ErrorCode& err_code, - Api::IoError::IoErrorCode err) -> void { - ASSERT_EQ(UdpListenerCallbacks::ErrorCode::SyscallError, err_code); + EXPECT_CALL(listener_callbacks_, onReceiveError(_)) + .WillOnce(Invoke([&](Api::IoError::IoErrorCode err) -> void { ASSERT_EQ(Api::IoError::IoErrorCode::NoSupport, err); dispatcher_->exit(); diff --git a/test/extensions/filters/udp/udp_proxy/BUILD b/test/extensions/filters/udp/udp_proxy/BUILD index 0a2c103376b9..f49ca2ff9a79 100644 --- a/test/extensions/filters/udp/udp_proxy/BUILD +++ b/test/extensions/filters/udp/udp_proxy/BUILD @@ -11,6 +11,18 @@ load( envoy_package() +envoy_extension_cc_test( + name = "udp_proxy_filter_test", + srcs = ["udp_proxy_filter_test.cc"], + extension_name = "envoy.filters.udp_listener.udp_proxy", + deps = [ + "//source/extensions/filters/udp/udp_proxy:udp_proxy_filter_lib", + "//test/mocks/network:io_handle_mocks", + "//test/mocks/upstream:upstream_mocks", + "@envoy_api//envoy/config/filter/udp/udp_proxy/v2alpha:pkg_cc_proto", + ], +) + envoy_extension_cc_test( name = "udp_proxy_integration_test", srcs = ["udp_proxy_integration_test.cc"], diff --git a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc new file mode 100644 index 000000000000..e82eee9931a0 --- /dev/null +++ b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc @@ -0,0 +1,248 @@ +#include "envoy/config/filter/udp/udp_proxy/v2alpha/udp_proxy.pb.validate.h" + +#include "extensions/filters/udp/udp_proxy/udp_proxy_filter.h" + +#include "test/mocks/network/io_handle.h" +#include "test/mocks/upstream/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::AtLeast; +using testing::ByMove; +using testing::InSequence; +using testing::Return; +using testing::SaveArg; + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace { + +class TestUdpProxyFilter : public UdpProxyFilter { +public: + using UdpProxyFilter::UdpProxyFilter; + + MOCK_METHOD1(createIoHandle, Network::IoHandlePtr(const Upstream::HostConstSharedPtr& host)); +}; + +Api::IoCallUint64Result makeNoError(uint64_t rc) { + auto no_error = Api::ioCallUint64ResultNoError(); + no_error.rc_ = rc; + return no_error; +} + +Api::IoCallUint64Result makeError(int sys_errno) { + return Api::IoCallUint64Result(0, Api::IoErrorPtr(new Network::IoSocketError(sys_errno), + Network::IoSocketError::deleteIoError)); +} + +class UdpProxyFilterTest : public testing::Test { +public: + struct TestSession { + TestSession(UdpProxyFilterTest& parent, + const Network::Address::InstanceConstSharedPtr& upstream_address) + : parent_(parent), upstream_address_(upstream_address), + io_handle_(new Network::MockIoHandle()) {} + + void expectUpstreamWrite(const std::string& data, int sys_errno = 0) { + EXPECT_CALL(*idle_timer_, enableTimer(parent_.config_->sessionTimeout(), nullptr)); + EXPECT_CALL(*io_handle_, sendmsg(_, 1, 0, nullptr, _)) + .WillOnce(Invoke( + [this, data, sys_errno]( + const Buffer::RawSlice* slices, uint64_t, int, const Network::Address::Ip*, + const Network::Address::Instance& peer_address) -> Api::IoCallUint64Result { + EXPECT_EQ(data, absl::string_view(static_cast(slices[0].mem_), + slices[0].len_)); + EXPECT_EQ(peer_address, *upstream_address_); + return sys_errno == 0 ? makeNoError(data.size()) : makeError(sys_errno); + })); + } + + void recvDataFromUpstream(const std::string& data, int send_sys_errno = 0) { + EXPECT_CALL(*idle_timer_, enableTimer(parent_.config_->sessionTimeout(), nullptr)); + + // Return the datagram. + EXPECT_CALL(*io_handle_, recvmsg(_, 1, _, _)) + .WillOnce(Invoke( + [this, data](Buffer::RawSlice* slices, const uint64_t, uint32_t, + Network::IoHandle::RecvMsgOutput& output) -> Api::IoCallUint64Result { + ASSERT(data.size() <= slices[0].len_); + memcpy(slices[0].mem_, data.data(), data.size()); + output.peer_address_ = upstream_address_; + return makeNoError(data.size()); + })); + // Send the datagram downstream. + EXPECT_CALL(parent_.callbacks_.udp_listener_, send(_)) + .WillOnce(Invoke([data, send_sys_errno]( + const Network::UdpSendData& send_data) -> Api::IoCallUint64Result { + // TODO(mattklein123): Verify peer/local address. + EXPECT_EQ(send_data.buffer_.toString(), data); + if (send_sys_errno == 0) { + send_data.buffer_.drain(send_data.buffer_.length()); + return makeNoError(data.size()); + } else { + return makeError(send_sys_errno); + } + })); + // Return an EAGAIN result. + EXPECT_CALL(*io_handle_, recvmsg(_, 1, _, _)) + .WillOnce(Return(ByMove(Api::IoCallUint64Result( + 0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), + Network::IoSocketError::deleteIoError))))); + // Kick off the receive. + file_event_cb_(Event::FileReadyType::Read); + } + + UdpProxyFilterTest& parent_; + const Network::Address::InstanceConstSharedPtr upstream_address_; + Event::MockTimer* idle_timer_{}; + Network::MockIoHandle* io_handle_; + Event::FileReadyCb file_event_cb_; + }; + + UdpProxyFilterTest() + : upstream_address_(Network::Utility::parseInternetAddressAndPort("20.0.0.1:443")) { + // Disable strict mock warnings. + EXPECT_CALL(callbacks_, udpListener()).Times(AtLeast(0)); + EXPECT_CALL(*cluster_manager_.thread_local_cluster_.lb_.host_, address()) + .WillRepeatedly(Return(upstream_address_)); + } + + ~UdpProxyFilterTest() { EXPECT_CALL(callbacks_.udp_listener_, onDestroy()); } + + void setup(const std::string& yaml) { + envoy::config::filter::udp::udp_proxy::v2alpha::UdpProxyConfig config; + TestUtility::loadFromYamlAndValidate(yaml, config); + config_ = std::make_shared(cluster_manager_, time_system_, stats_store_, + config); + filter_ = std::make_unique(callbacks_, config_); + } + + void recvDataFromDownstream(const std::string& peer_address, const std::string& local_address, + const std::string& buffer) { + Network::UdpRecvData data; + data.addresses_.peer_ = Network::Utility::parseInternetAddressAndPort(peer_address); + data.addresses_.local_ = Network::Utility::parseInternetAddressAndPort(local_address); + data.buffer_ = std::make_unique(buffer); + data.receive_time_ = MonotonicTime(std::chrono::seconds(0)); + filter_->onData(data); + } + + void expectSessionCreate() { + test_sessions_.emplace_back(*this, upstream_address_); + TestSession& new_session = test_sessions_.back(); + EXPECT_CALL(cluster_manager_, get(_)); + new_session.idle_timer_ = new Event::MockTimer(&callbacks_.udp_listener_.dispatcher_); + EXPECT_CALL(*filter_, createIoHandle(_)) + .WillOnce(Return(ByMove(Network::IoHandlePtr{test_sessions_.back().io_handle_}))); + EXPECT_CALL(*new_session.io_handle_, fd()); + EXPECT_CALL(callbacks_.udp_listener_.dispatcher_, + createFileEvent_(_, _, Event::FileTriggerType::Edge, Event::FileReadyType::Read)) + .WillOnce(DoAll(SaveArg<1>(&new_session.file_event_cb_), Return(nullptr))); + } + + void checkTransferStats(uint64_t rx_bytes, uint64_t rx_datagrams, uint64_t tx_bytes, + uint64_t tx_datagrams) { + EXPECT_EQ(rx_bytes, config_->stats().downstream_sess_rx_bytes_.value()); + EXPECT_EQ(rx_datagrams, config_->stats().downstream_sess_rx_datagrams_.value()); + EXPECT_EQ(tx_bytes, config_->stats().downstream_sess_tx_bytes_.value()); + EXPECT_EQ(tx_datagrams, config_->stats().downstream_sess_tx_datagrams_.value()); + } + + Upstream::MockClusterManager cluster_manager_; + NiceMock time_system_; + Stats::IsolatedStoreImpl stats_store_; + UdpProxyFilterConfigSharedPtr config_; + Network::MockUdpReadFilterCallbacks callbacks_; + std::unique_ptr filter_; + std::vector test_sessions_; + // If a test ever support more than 1 upstream host this will need to move to the session/test. + const Network::Address::InstanceConstSharedPtr upstream_address_; +}; + +// Basic UDP proxy flow with a single session. +TEST_F(UdpProxyFilterTest, BasicFlow) { + InSequence s; + + setup(R"EOF( +stat_prefix: foo +cluster: fake_cluster + )EOF"); + + expectSessionCreate(); + test_sessions_[0].expectUpstreamWrite("hello"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); + checkTransferStats(5 /*rx_bytes*/, 1 /*rx_datagrams*/, 0 /*tx_bytes*/, 0 /*tx_datagrams*/); + test_sessions_[0].recvDataFromUpstream("world"); + checkTransferStats(5 /*rx_bytes*/, 1 /*rx_datagrams*/, 5 /*tx_bytes*/, 1 /*tx_datagrams*/); + + test_sessions_[0].expectUpstreamWrite("hello2"); + test_sessions_[0].expectUpstreamWrite("hello3"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello2"); + checkTransferStats(11 /*rx_bytes*/, 2 /*rx_datagrams*/, 5 /*tx_bytes*/, 1 /*tx_datagrams*/); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello3"); + checkTransferStats(17 /*rx_bytes*/, 3 /*rx_datagrams*/, 5 /*tx_bytes*/, 1 /*tx_datagrams*/); + + test_sessions_[0].recvDataFromUpstream("world2"); + checkTransferStats(17 /*rx_bytes*/, 3 /*rx_datagrams*/, 11 /*tx_bytes*/, 2 /*tx_datagrams*/); + test_sessions_[0].recvDataFromUpstream("world3"); + checkTransferStats(17 /*rx_bytes*/, 3 /*rx_datagrams*/, 17 /*tx_bytes*/, 3 /*tx_datagrams*/); +} + +// Idle timeout flow. +TEST_F(UdpProxyFilterTest, IdleTimeout) { + InSequence s; + + setup(R"EOF( +stat_prefix: foo +cluster: fake_cluster + )EOF"); + + expectSessionCreate(); + test_sessions_[0].expectUpstreamWrite("hello"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); + + test_sessions_[0].idle_timer_->invokeCallback(); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(0, config_->stats().downstream_sess_active_.value()); + + expectSessionCreate(); + test_sessions_[1].expectUpstreamWrite("hello"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + EXPECT_EQ(2, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); +} + +// Verify downstream send and receive error handling. +TEST_F(UdpProxyFilterTest, SendReceiveErrorHandling) { + InSequence s; + + setup(R"EOF( +stat_prefix: foo +cluster: fake_cluster + )EOF"); + + filter_->onReceiveError(Api::IoError::IoErrorCode::UnknownError); + EXPECT_EQ(1, config_->stats().downstream_sess_rx_errors_.value()); + + expectSessionCreate(); + test_sessions_[0].expectUpstreamWrite("hello"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + checkTransferStats(5 /*rx_bytes*/, 1 /*rx_datagrams*/, 0 /*tx_bytes*/, 0 /*tx_datagrams*/); + + test_sessions_[0].recvDataFromUpstream("world2", EMSGSIZE); + checkTransferStats(5 /*rx_bytes*/, 1 /*rx_datagrams*/, 0 /*tx_bytes*/, 0 /*tx_datagrams*/); + EXPECT_EQ(1, config_->stats().downstream_sess_tx_errors_.value()); +} + +} // namespace +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/udp/udp_proxy/udp_proxy_integration_test.cc b/test/extensions/filters/udp/udp_proxy/udp_proxy_integration_test.cc index b644e3d26115..727b21ab0870 100644 --- a/test/extensions/filters/udp/udp_proxy/udp_proxy_integration_test.cc +++ b/test/extensions/filters/udp/udp_proxy/udp_proxy_integration_test.cc @@ -23,6 +23,7 @@ class UdpSyncClient { } void recv(Network::UdpRecvData& datagram) { + datagram = Network::UdpRecvData(); const auto rc = Network::Test::readFromSocket(socket_->ioHandle(), *socket_->localAddress(), datagram); ASSERT_TRUE(rc.ok()); @@ -43,6 +44,7 @@ class UdpProxyIntegrationTest : public testing::TestWithParamtoString()); // Respond from the upstream. - fake_upstreams_[0]->sendUdpDatagram("world", *request_datagram.addresses_.peer_); + fake_upstreams_[0]->sendUdpDatagram("world1", *request_datagram.addresses_.peer_); Network::UdpRecvData response_datagram; client.recv(response_datagram); - EXPECT_EQ("world", response_datagram.buffer_->toString()); + EXPECT_EQ("world1", response_datagram.buffer_->toString()); EXPECT_EQ(listener_address.asString(), response_datagram.addresses_.peer_->asString()); + + EXPECT_EQ(5, test_server_->counter("udp.foo.downstream_sess_rx_bytes")->value()); + EXPECT_EQ(1, test_server_->counter("udp.foo.downstream_sess_rx_datagrams")->value()); + EXPECT_EQ(6, test_server_->counter("udp.foo.downstream_sess_tx_bytes")->value()); + EXPECT_EQ(1, test_server_->counter("udp.foo.downstream_sess_tx_datagrams")->value()); + EXPECT_EQ(1, test_server_->counter("udp.foo.downstream_sess_total")->value()); + EXPECT_EQ(1, test_server_->gauge("udp.foo.downstream_sess_active")->value()); } }; diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 997977f5892c..9a7c0edf340a 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -627,6 +627,7 @@ class FakeUpstream : Logger::Loggable, // Network::UdpListenerReadFilter void onData(Network::UdpRecvData& data) override { parent_.onRecvDatagram(data); } + void onReceiveError(Api::IoError::IoErrorCode) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } private: FakeUpstream& parent_; diff --git a/test/mocks/network/BUILD b/test/mocks/network/BUILD index 6011484c1094..646a5ee1a172 100644 --- a/test/mocks/network/BUILD +++ b/test/mocks/network/BUILD @@ -20,6 +20,15 @@ envoy_cc_mock( ], ) +envoy_cc_mock( + name = "io_handle_mocks", + srcs = ["io_handle.cc"], + hdrs = ["io_handle.h"], + deps = [ + "//include/envoy/network:io_handle_interface", + ], +) + envoy_cc_mock( name = "network_mocks", srcs = ["mocks.cc"], diff --git a/test/mocks/network/io_handle.cc b/test/mocks/network/io_handle.cc new file mode 100644 index 000000000000..3f5871626651 --- /dev/null +++ b/test/mocks/network/io_handle.cc @@ -0,0 +1,12 @@ +#include "test/mocks/network/io_handle.h" + +#include "envoy/network/address.h" + +namespace Envoy { +namespace Network { + +MockIoHandle::MockIoHandle() = default; +MockIoHandle::~MockIoHandle() = default; + +} // namespace Network +} // namespace Envoy diff --git a/test/mocks/network/io_handle.h b/test/mocks/network/io_handle.h new file mode 100644 index 000000000000..b87bded3dc16 --- /dev/null +++ b/test/mocks/network/io_handle.h @@ -0,0 +1,29 @@ +#pragma once + +#include "envoy/network/io_handle.h" + +#include "gmock/gmock.h" + +namespace Envoy { +namespace Network { + +class MockIoHandle : public IoHandle { +public: + MockIoHandle(); + ~MockIoHandle(); + + MOCK_CONST_METHOD0(fd, int()); + MOCK_METHOD0(close, Api::IoCallUint64Result()); + MOCK_CONST_METHOD0(isOpen, bool()); + MOCK_METHOD3(readv, Api::IoCallUint64Result(uint64_t max_length, Buffer::RawSlice* slices, + uint64_t num_slice)); + MOCK_METHOD2(writev, Api::IoCallUint64Result(const Buffer::RawSlice* slices, uint64_t num_slice)); + MOCK_METHOD5(sendmsg, Api::IoCallUint64Result(const Buffer::RawSlice* slices, uint64_t num_slice, + int flags, const Address::Ip* self_ip, + const Address::Instance& peer_address)); + MOCK_METHOD4(recvmsg, Api::IoCallUint64Result(Buffer::RawSlice* slices, const uint64_t num_slice, + uint32_t self_port, RecvMsgOutput& output)); +}; + +} // namespace Network +} // namespace Envoy diff --git a/test/mocks/network/mocks.cc b/test/mocks/network/mocks.cc index 578e928483c5..fd20cb5e5b11 100644 --- a/test/mocks/network/mocks.cc +++ b/test/mocks/network/mocks.cc @@ -172,7 +172,10 @@ MockTransportSocketCallbacks::MockTransportSocketCallbacks() { } MockTransportSocketCallbacks::~MockTransportSocketCallbacks() = default; -MockUdpListener::MockUdpListener() = default; +MockUdpListener::MockUdpListener() { + ON_CALL(*this, dispatcher()).WillByDefault(ReturnRef(dispatcher_)); +} + MockUdpListener::~MockUdpListener() { onDestroy(); } MockUdpReadFilterCallbacks::MockUdpReadFilterCallbacks() { diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index bf85ab789c63..bc545d330634 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -136,19 +136,9 @@ class MockUdpListenerCallbacks : public UdpListenerCallbacks { MockUdpListenerCallbacks(); ~MockUdpListenerCallbacks() override; - void onData(UdpRecvData& data) override { onData_(data); } - - void onWriteReady(const Socket& socket) override { onWriteReady_(socket); } - - void onReceiveError(const ErrorCode& err_code, Api::IoError::IoErrorCode err) override { - onReceiveError_(err_code, err); - } - - MOCK_METHOD1(onData_, void(UdpRecvData& data)); - - MOCK_METHOD1(onWriteReady_, void(const Socket& socket)); - - MOCK_METHOD2(onReceiveError_, void(const ErrorCode& err_code, Api::IoError::IoErrorCode err)); + MOCK_METHOD1(onData, void(UdpRecvData& data)); + MOCK_METHOD1(onWriteReady, void(const Socket& socket)); + MOCK_METHOD1(onReceiveError, void(Api::IoError::IoErrorCode err)); }; class MockDrainDecision : public DrainDecision { @@ -428,6 +418,8 @@ class MockUdpListener : public UdpListener { MOCK_METHOD0(dispatcher, Event::Dispatcher&()); MOCK_CONST_METHOD0(localAddress, Address::InstanceConstSharedPtr&()); MOCK_METHOD1(send, Api::IoCallUint64Result(const UdpSendData&)); + + Event::MockDispatcher dispatcher_; }; class MockUdpReadFilterCallbacks : public UdpReadFilterCallbacks { diff --git a/test/test_common/network_utility.cc b/test/test_common/network_utility.cc index 6f50bb5465bb..207925f1ca16 100644 --- a/test/test_common/network_utility.cc +++ b/test/test_common/network_utility.cc @@ -191,7 +191,7 @@ const Network::FilterChainSharedPtr createEmptyFilterChainWithRawBufferSockets() namespace { struct SyncPacketProcessor : public Network::UdpPacketProcessor { - SyncPacketProcessor(Network::UdpRecvData& data) : data_(data) {} + SyncPacketProcessor(Network::UdpRecvData& data) : data_(data) { ASSERT(data.buffer_ == nullptr); } void processPacket(Network::Address::InstanceConstSharedPtr local_address, Network::Address::InstanceConstSharedPtr peer_address, diff --git a/test/test_common/network_utility.h b/test/test_common/network_utility.h index 9924a29e54e3..a3410bf7003c 100644 --- a/test/test_common/network_utility.h +++ b/test/test_common/network_utility.h @@ -162,7 +162,8 @@ const FilterChainSharedPtr createEmptyFilterChainWithRawBufferSockets(); /** * Wrapper for Utility::readFromSocket() which reads a single datagram into the supplied - * UdpRecvData without worrying about the packet processor interface. + * UdpRecvData without worrying about the packet processor interface. The function will + * instantiate the buffer returned in data. */ Api::IoCallUint64Result readFromSocket(IoHandle& handle, const Address::Instance& local_address, UdpRecvData& data);