From bb5b9b46cbc02b4b9ac1a16eaa77c609d1aa1802 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Tue, 26 Jan 2021 12:51:40 -0500 Subject: [PATCH 1/8] quic: HTTP/3 upstream prototype Signed-off-by: Alyssa Wilk --- include/envoy/upstream/cluster_manager.h | 2 +- include/envoy/upstream/upstream.h | 1 + source/common/http/conn_pool_base.cc | 3 +- source/common/http/http2/conn_pool.cc | 35 ++++--- source/common/http/http2/conn_pool.h | 32 ++++-- source/common/http/http3/BUILD | 15 +++ source/common/http/http3/conn_pool.cc | 57 +++++++++++ source/common/http/http3/conn_pool.h | 32 ++++++ source/common/http/http3/quic_codec_factory.h | 40 ++++++++ source/common/upstream/BUILD | 1 + .../common/upstream/cluster_manager_impl.cc | 10 +- source/common/upstream/cluster_manager_impl.h | 2 +- source/common/upstream/upstream_impl.cc | 4 - source/extensions/quic_listeners/quiche/BUILD | 7 +- .../quiche/active_quic_listener.cc | 10 +- .../quiche/active_quic_listener.h | 2 +- .../quic_listeners/quiche/codec_impl.cc | 3 + .../quic_listeners/quiche/codec_impl.h | 97 +++++++++++++++++++ .../quic_filter_manager_connection_impl.h | 6 +- .../transport_sockets/tls/ssl_socket.h | 2 + test/common/upstream/test_cluster_manager.h | 2 +- test/common/upstream/upstream_impl_test.cc | 19 ++-- test/config/utility.cc | 2 + .../quiche/active_quic_listener_test.cc | 2 +- .../quic_listeners/quiche/integration/BUILD | 19 ++++ .../quic_protocol_integration_test.cc | 15 +++ test/integration/BUILD | 2 + test/integration/base_integration_test.cc | 28 +++++- test/integration/fake_upstream.cc | 70 ++++++++++--- test/integration/fake_upstream.h | 30 ++++-- test/integration/http_protocol_integration.cc | 23 +++-- test/integration/protocol_integration_test.cc | 32 ++++-- test/mocks/upstream/cluster_manager_factory.h | 2 +- 33 files changed, 520 insertions(+), 87 deletions(-) create mode 100644 source/common/http/http3/conn_pool.cc create mode 100644 source/common/http/http3/conn_pool.h create mode 100644 test/extensions/quic_listeners/quiche/integration/quic_protocol_integration_test.cc diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index ecaf617b3c70..825142e5a3ba 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -356,7 +356,7 @@ class ClusterManagerFactory { ResourcePriority priority, std::vector& protocol, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsSharedPtr& transport_socket_options, - ClusterConnectivityState& state) PURE; + TimeSource& time_source, ClusterConnectivityState& state) PURE; /** * Allocate a TCP connection pool for the host. Pools are separated by 'priority' and diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index a784c35b5498..cd66241e5a31 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -539,6 +539,7 @@ class PrioritySet { COUNTER(upstream_cx_destroy_with_active_rq) \ COUNTER(upstream_cx_http1_total) \ COUNTER(upstream_cx_http2_total) \ + COUNTER(upstream_cx_http3_total) \ COUNTER(upstream_cx_idle_timeout) \ COUNTER(upstream_cx_max_requests) \ COUNTER(upstream_cx_none_healthy) \ diff --git a/source/common/http/conn_pool_base.cc b/source/common/http/conn_pool_base.cc index 19c057a53edb..87f096440075 100644 --- a/source/common/http/conn_pool_base.cc +++ b/source/common/http/conn_pool_base.cc @@ -31,8 +31,7 @@ wrapTransportSocketOptions(Network::TransportSocketOptionsSharedPtr transport_so fallbacks.push_back(Http::Utility::AlpnNames::get().Http2); break; case Http::Protocol::Http3: - // TODO(snowp): Add once HTTP/3 upstream support is added. - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + // TODO(#14829) hard-code H3 ALPN, consider failing if other things are negotiated. break; } } diff --git a/source/common/http/http2/conn_pool.cc b/source/common/http/http2/conn_pool.cc index 93af8a2c8fcb..00d79eb3a762 100644 --- a/source/common/http/http2/conn_pool.cc +++ b/source/common/http/http2/conn_pool.cc @@ -10,13 +10,12 @@ namespace Envoy { namespace Http { -namespace Http2 { // All streams are 2^31. Client streams are half that, minus stream 0. Just to be on the safe // side we do 2^29. static const uint64_t DEFAULT_MAX_STREAMS = (1 << 29); -void ActiveClient::onGoAway(Http::GoAwayErrorCode) { +void MultiplexedActiveClientBase::onGoAway(Http::GoAwayErrorCode) { ENVOY_CONN_LOG(debug, "remote goaway", *codec_client_); parent_.host()->cluster().stats().upstream_cx_close_notify_.inc(); if (state_ != ActiveClient::State::DRAINING) { @@ -28,7 +27,7 @@ void ActiveClient::onGoAway(Http::GoAwayErrorCode) { } } -void ActiveClient::onStreamDestroy() { +void MultiplexedActiveClientBase::onStreamDestroy() { parent().onStreamClosed(*this, false); // If we are destroying this stream because of a disconnect, do not check for drain here. We will @@ -39,7 +38,7 @@ void ActiveClient::onStreamDestroy() { } } -void ActiveClient::onStreamReset(Http::StreamResetReason reason) { +void MultiplexedActiveClientBase::onStreamReset(Http::StreamResetReason reason) { if (reason == StreamResetReason::ConnectionTermination || reason == StreamResetReason::ConnectionFailure) { parent_.host()->cluster().stats().upstream_rq_pending_failure_eject_.inc(); @@ -55,31 +54,45 @@ uint64_t maxStreamsPerConnection(uint64_t max_streams_config) { return (max_streams_config != 0) ? max_streams_config : DEFAULT_MAX_STREAMS; } -ActiveClient::ActiveClient(HttpConnPoolImplBase& parent) +MultiplexedActiveClientBase::MultiplexedActiveClientBase(HttpConnPoolImplBase& parent, + Stats::Counter& cx_total) : Envoy::Http::ActiveClient( parent, maxStreamsPerConnection(parent.host()->cluster().maxRequestsPerConnection()), parent.host()->cluster().http2Options().max_concurrent_streams().value()) { codec_client_->setCodecClientCallbacks(*this); codec_client_->setCodecConnectionCallbacks(*this); - parent.host()->cluster().stats().upstream_cx_http2_total_.inc(); + cx_total.inc(); } -ActiveClient::ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent, - Upstream::Host::CreateConnectionData& data) +MultiplexedActiveClientBase::MultiplexedActiveClientBase(Envoy::Http::HttpConnPoolImplBase& parent, + Upstream::Host::CreateConnectionData& data, + Stats::Counter& cx_total) : Envoy::Http::ActiveClient( parent, maxStreamsPerConnection(parent.host()->cluster().maxRequestsPerConnection()), parent.host()->cluster().http2Options().max_concurrent_streams().value(), data) { codec_client_->setCodecClientCallbacks(*this); codec_client_->setCodecConnectionCallbacks(*this); - parent.host()->cluster().stats().upstream_cx_http2_total_.inc(); + cx_total.inc(); } -bool ActiveClient::closingWithIncompleteStream() const { return closed_with_active_rq_; } +bool MultiplexedActiveClientBase::closingWithIncompleteStream() const { + return closed_with_active_rq_; +} -RequestEncoder& ActiveClient::newStreamEncoder(ResponseDecoder& response_decoder) { +RequestEncoder& MultiplexedActiveClientBase::newStreamEncoder(ResponseDecoder& response_decoder) { return codec_client_->newStream(response_decoder); } +namespace Http2 { +ActiveClient::ActiveClient(HttpConnPoolImplBase& parent) + : MultiplexedActiveClientBase(parent, + parent.host()->cluster().stats().upstream_cx_http2_total_) {} + +ActiveClient::ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent, + Upstream::Host::CreateConnectionData& data) + : MultiplexedActiveClientBase(parent, data, + parent.host()->cluster().stats().upstream_cx_http2_total_) {} + ConnectionPool::InstancePtr allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator, Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority, diff --git a/source/common/http/http2/conn_pool.h b/source/common/http/http2/conn_pool.h index c4c1b14801e7..51a25cbc051e 100644 --- a/source/common/http/http2/conn_pool.h +++ b/source/common/http/http2/conn_pool.h @@ -9,19 +9,16 @@ namespace Envoy { namespace Http { -namespace Http2 { /** - * Implementation of an active client for HTTP/2 + * Active client base for HTTP/2 and HTTP/3 */ -class ActiveClient : public CodecClientCallbacks, - public Http::ConnectionCallbacks, - public Envoy::Http::ActiveClient { +class MultiplexedActiveClientBase : public CodecClientCallbacks, + public Http::ConnectionCallbacks, + public Envoy::Http::ActiveClient { public: - ActiveClient(HttpConnPoolImplBase& parent); - ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent, - Upstream::Host::CreateConnectionData& data); - ~ActiveClient() override = default; + MultiplexedActiveClientBase(HttpConnPoolImplBase& parent, Stats::Counter& cx_total); + ~MultiplexedActiveClientBase() override = default; // ConnPoolImpl::ActiveClient bool closingWithIncompleteStream() const override; @@ -34,9 +31,26 @@ class ActiveClient : public CodecClientCallbacks, // Http::ConnectionCallbacks void onGoAway(Http::GoAwayErrorCode error_code) override; +protected: + MultiplexedActiveClientBase(Envoy::Http::HttpConnPoolImplBase& parent, + Upstream::Host::CreateConnectionData& data, Stats::Counter& cx_total); + +private: bool closed_with_active_rq_{}; }; +namespace Http2 { + +/** + * Implementation of an active client for HTTP/2 + */ +class ActiveClient : public MultiplexedActiveClientBase { +public: + ActiveClient(HttpConnPoolImplBase& parent); + ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent, + Upstream::Host::CreateConnectionData& data); +}; + ConnectionPool::InstancePtr allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator, Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority, diff --git a/source/common/http/http3/BUILD b/source/common/http/http3/BUILD index 43ba5729097c..6451b69c3f97 100644 --- a/source/common/http/http3/BUILD +++ b/source/common/http/http3/BUILD @@ -8,6 +8,19 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_library( + name = "conn_pool_lib", + srcs = ["conn_pool.cc"], + hdrs = ["conn_pool.h"], + deps = [ + "//include/envoy/event:dispatcher_interface", + "//include/envoy/upstream:upstream_interface", + "//source/common/http:codec_client_lib", + "//source/common/http:conn_pool_base_lib", + "//source/common/http/http2:conn_pool_lib", + ], +) + envoy_cc_library( name = "quic_codec_factory_lib", hdrs = ["quic_codec_factory.h"], @@ -15,6 +28,8 @@ envoy_cc_library( "//include/envoy/config:typed_config_interface", "//include/envoy/http:codec_interface", "//include/envoy/network:connection_interface", + "//include/envoy/ssl:context_config_interface", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", ], ) diff --git a/source/common/http/http3/conn_pool.cc b/source/common/http/http3/conn_pool.cc new file mode 100644 index 000000000000..f4eaf22e2bb1 --- /dev/null +++ b/source/common/http/http3/conn_pool.cc @@ -0,0 +1,57 @@ +#include "common/http/http3/conn_pool.h" + +#include + +#include "envoy/event/dispatcher.h" +#include "envoy/upstream/upstream.h" + +#include "common/config/utility.h" +#include "common/http/http3/quic_codec_factory.h" +#include "common/http/http3/well_known_names.h" +#include "common/http/utility.h" +#include "common/network/address_impl.h" +#include "common/network/utility.h" +#include "common/runtime/runtime_features.h" + +namespace Envoy { +namespace Http { +namespace Http3 { + +ConnectionPool::InstancePtr +allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator, + Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority, + const Network::ConnectionSocket::OptionsSharedPtr& options, + const Network::TransportSocketOptionsSharedPtr& transport_socket_options, + Upstream::ClusterConnectivityState& state, TimeSource& time_source) { + return std::make_unique( + host, priority, dispatcher, options, transport_socket_options, random_generator, state, + [](HttpConnPoolImplBase* pool) { return std::make_unique(*pool); }, + [&dispatcher, &time_source](Upstream::Host::CreateConnectionData& data, + HttpConnPoolImplBase* pool) { + // TODO(#14829) this is creating CreateConnectionData and overwriting. Refactor the base + // pool class to inject the correct CreateConnectionData and avoid the wasted allocation. + auto host_address = data.host_description_->address(); + auto source_address = data.host_description_->cluster().sourceAddress(); + if (!source_address.get()) { + source_address = Network::Utility::getLocalAddress(host_address->ip()->version()); + } + Network::TransportSocketFactory& transport_socket_factory = + data.host_description_->transportSocketFactory(); + data.connection_->close(Network::ConnectionCloseType::NoFlush); + data.connection_ = + Config::Utility::getAndCheckFactoryByName( + Http::QuicCodecNames::get().Quiche) + .createQuicNetworkConnection(host_address, source_address, transport_socket_factory, + data.host_description_->cluster().statsScope(), + dispatcher, time_source); + CodecClientPtr codec{new CodecClientProd( + CodecClient::Type::HTTP3, std::move(data.connection_), data.host_description_, + pool->dispatcher(), pool->randomGenerator())}; + return codec; + }, + std::vector{Protocol::Http3}); +} + +} // namespace Http3 +} // namespace Http +} // namespace Envoy diff --git a/source/common/http/http3/conn_pool.h b/source/common/http/http3/conn_pool.h new file mode 100644 index 000000000000..988cc176a969 --- /dev/null +++ b/source/common/http/http3/conn_pool.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include "envoy/upstream/upstream.h" + +#include "common/http/codec_client.h" +#include "common/http/http2/conn_pool.h" + +namespace Envoy { +namespace Http { +namespace Http3 { + +// TODO(#14829) the constructor of Http2::ActiveClient sets max requests per +// connection based on HTTP/2 config. Sort out the HTTP/3 config story. +class ActiveClient : public MultiplexedActiveClientBase { +public: + ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent) + : MultiplexedActiveClientBase(parent, + parent.host()->cluster().stats().upstream_cx_http3_total_) {} +}; + +ConnectionPool::InstancePtr +allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator, + Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority, + const Network::ConnectionSocket::OptionsSharedPtr& options, + const Network::TransportSocketOptionsSharedPtr& transport_socket_options, + Upstream::ClusterConnectivityState& state, TimeSource& time_source); + +} // namespace Http3 +} // namespace Http +} // namespace Envoy diff --git a/source/common/http/http3/quic_codec_factory.h b/source/common/http/http3/quic_codec_factory.h index 0b4a72404200..030362d28387 100644 --- a/source/common/http/http3/quic_codec_factory.h +++ b/source/common/http/http3/quic_codec_factory.h @@ -2,9 +2,11 @@ #include +#include "envoy/config/listener/v3/quic_config.pb.h" #include "envoy/config/typed_config.h" #include "envoy/http/codec.h" #include "envoy/network/connection.h" +#include "envoy/ssl/context_config.h" namespace Envoy { namespace Http { @@ -31,5 +33,43 @@ class QuicHttpClientConnectionFactory : public Config::UntypedFactory { std::string category() const override { return "envoy.quic_server_codec"; } }; +// A factory to create EnvoyQuicClientConnection instance for QUIC +class QuicClientConnectionFactory : public Config::UntypedFactory { +public: + ~QuicClientConnectionFactory() override = default; + + virtual std::unique_ptr + createQuicNetworkConnection(Network::Address::InstanceConstSharedPtr server_addr, + Network::Address::InstanceConstSharedPtr local_addr, + Network::TransportSocketFactory& transport_socket_factory, + Stats::Scope& stats_scope, Event::Dispatcher& dispatcher, + TimeSource& time_source) PURE; + + std::string category() const override { return "envoy.quic_connection"; } +}; + +// A factory to create ActiveQuicListenerFactory instance for QUIC +class QuicActiveQuicListenerFactory : public Config::UntypedFactory { +public: + ~QuicActiveQuicListenerFactory() override = default; + + virtual Network::ActiveUdpListenerFactoryPtr + createActiveQuicListener(envoy::config::listener::v3::QuicProtocolOptions, + uint32_t concurrency) PURE; + + std::string category() const override { return "envoy.quic_listener"; } +}; + +// A factory to create QuicServerTransportSocketFactory instance for QUIC +class QuicServerTransportSocketFactory : public Config::UntypedFactory { +public: + ~QuicServerTransportSocketFactory() override = default; + + virtual Network::TransportSocketFactoryPtr + createQuicServerTransportSocketFactory(Ssl::ServerContextConfigPtr config) PURE; + + std::string category() const override { return "envoy.quic_server_transport_factory"; } +}; + } // namespace Http } // namespace Envoy diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 318e0c285d03..8ae9aa9b11e4 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -61,6 +61,7 @@ envoy_cc_library( "//source/common/http:mixed_conn_pool", "//source/common/http/http1:conn_pool_lib", "//source/common/http/http2:conn_pool_lib", + "//source/common/http/http3:conn_pool_lib", "//source/common/network:resolver_lib", "//source/common/network:utility_lib", "//source/common/protobuf:utility_lib", diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index e9af3f815dd9..59efe0360ad8 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -28,6 +28,7 @@ #include "common/http/async_client_impl.h" #include "common/http/http1/conn_pool.h" #include "common/http/http2/conn_pool.h" +#include "common/http/http3/conn_pool.h" #include "common/http/mixed_conn_pool.h" #include "common/network/resolver_impl.h" #include "common/network/utility.h" @@ -1419,7 +1420,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::connPool( parent_.thread_local_dispatcher_, host, priority, upstream_protocols, !upstream_options->empty() ? upstream_options : nullptr, have_transport_socket_options ? context->upstreamTransportSocketOptions() : nullptr, - parent_.cluster_manager_state_); + parent_.parent_.time_source_, parent_.cluster_manager_state_); }); if (pool.has_value()) { @@ -1488,7 +1489,7 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( Event::Dispatcher& dispatcher, HostConstSharedPtr host, ResourcePriority priority, std::vector& protocols, const Network::ConnectionSocket::OptionsSharedPtr& options, - const Network::TransportSocketOptionsSharedPtr& transport_socket_options, + const Network::TransportSocketOptionsSharedPtr& transport_socket_options, TimeSource& source, ClusterConnectivityState& state) { if (protocols.size() == 2) { ASSERT((protocols[0] == Http::Protocol::Http2 && protocols[1] == Http::Protocol::Http11) || @@ -1503,6 +1504,11 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( return Http::Http2::allocateConnPool(dispatcher, api_.randomGenerator(), host, priority, options, transport_socket_options, state); } + if (protocols.size() == 1 && protocols[0] == Http::Protocol::Http3 && + runtime_.snapshot().featureEnabled("upstream.use_http3", 100)) { + return Http::Http3::allocateConnPool(dispatcher, api_.randomGenerator(), host, priority, + options, transport_socket_options, state, source); + } ASSERT(protocols.size() == 1 && protocols[0] == Http::Protocol::Http11); return Http::Http1::allocateConnPool(dispatcher, api_.randomGenerator(), host, priority, options, transport_socket_options, state); diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 98d04ac2d734..f3ad2d6338de 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -64,7 +64,7 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { ResourcePriority priority, std::vector& protocol, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsSharedPtr& transport_socket_options, - ClusterConnectivityState& state) override; + TimeSource& time_source, ClusterConnectivityState& state) override; Tcp::ConnectionPool::InstancePtr allocateTcpConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host, ResourcePriority priority, diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 747d4079fcde..06dc755a7010 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -933,10 +933,6 @@ ClusterImplBase::ClusterImplBase( fmt::format("ALPN configured for cluster {} which has a non-ALPN transport socket: {}", cluster.name(), cluster.DebugString())); } - if ((info_->features() & ClusterInfoImpl::Features::HTTP3)) { - throw EnvoyException( - fmt::format("HTTP3 not yet supported: {}", cluster.name(), cluster.DebugString())); - } // Create the default (empty) priority set before registering callbacks to // avoid getting an update the first time it is accessed. diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 6dd9192114b0..75a5f9a08656 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -141,12 +141,18 @@ envoy_cc_library( hdrs = ["codec_impl.h"], tags = ["nofips"], deps = [ + ":active_quic_listener_lib", + ":envoy_quic_alarm_factory_lib", ":envoy_quic_client_session_lib", + ":envoy_quic_connection_helper_lib", + ":envoy_quic_proof_verifier_lib", ":envoy_quic_server_session_lib", + ":envoy_quic_utils_lib", "//include/envoy/http:codec_interface", "//include/envoy/registry", "//source/common/http/http3:quic_codec_factory_lib", "//source/common/http/http3:well_known_names", + "//source/extensions/transport_sockets/tls:ssl_socket_lib", "@com_googlesource_quiche//:quic_core_http_spdy_session_lib", ], ) @@ -377,7 +383,6 @@ envoy_cc_extension( ), security_posture = "unknown", tags = ["nofips"], - # QUICHE can't build against FIPS BoringSSL until the FIPS build # is on a new enough version to have QUIC support. Remove it from # the build until then. Re-enable as part of #7433. diff --git a/source/extensions/quic_listeners/quiche/active_quic_listener.cc b/source/extensions/quic_listeners/quiche/active_quic_listener.cc index 86bb75a2ed51..7a3843865246 100644 --- a/source/extensions/quic_listeners/quiche/active_quic_listener.cc +++ b/source/extensions/quic_listeners/quiche/active_quic_listener.cc @@ -38,8 +38,10 @@ ActiveQuicListener::ActiveQuicListener( dispatcher.createUdpListener(listen_socket, *this), &listener_config), dispatcher_(dispatcher), version_manager_(quic::CurrentSupportedVersions()), - kernel_worker_routing_(kernel_worker_routing), - enabled_(enabled, Runtime::LoaderSingleton::get()) { + kernel_worker_routing_(kernel_worker_routing) { + if (Runtime::LoaderSingleton::getExisting()) { + enabled_.emplace(Runtime::FeatureFlag(enabled, Runtime::LoaderSingleton::get())); + } if (options != nullptr) { const bool ok = Network::Socket::applyOptions( options, listen_socket_, envoy::config::core::v3::SocketOption::STATE_BOUND); @@ -97,7 +99,7 @@ void ActiveQuicListener::onListenerShutdown() { } void ActiveQuicListener::onDataWorker(Network::UdpRecvData&& data) { - if (!enabled_.enabled()) { + if (enabled_.has_value() && !enabled_.value().enabled()) { return; } @@ -127,7 +129,7 @@ void ActiveQuicListener::onDataWorker(Network::UdpRecvData&& data) { } void ActiveQuicListener::onReadReady() { - if (!enabled_.enabled()) { + if (enabled_.has_value() && !enabled_.value().enabled()) { ENVOY_LOG(trace, "Quic listener {}: runtime disabled", config_->name()); return; } diff --git a/source/extensions/quic_listeners/quiche/active_quic_listener.h b/source/extensions/quic_listeners/quiche/active_quic_listener.h index 878032406ea9..25a4588e279c 100644 --- a/source/extensions/quic_listeners/quiche/active_quic_listener.h +++ b/source/extensions/quic_listeners/quiche/active_quic_listener.h @@ -67,7 +67,7 @@ class ActiveQuicListener : public Envoy::Server::ActiveUdpListenerBase, quic::QuicVersionManager version_manager_; std::unique_ptr quic_dispatcher_; const bool kernel_worker_routing_; - Runtime::FeatureFlag enabled_; + absl::optional enabled_{}; Network::UdpPacketWriter* udp_packet_writer_; // The number of runs of the event loop in which at least one CHLO was buffered. diff --git a/source/extensions/quic_listeners/quiche/codec_impl.cc b/source/extensions/quic_listeners/quiche/codec_impl.cc index 51fad9e4bc20..75aede9d3dea 100644 --- a/source/extensions/quic_listeners/quiche/codec_impl.cc +++ b/source/extensions/quic_listeners/quiche/codec_impl.cc @@ -113,6 +113,9 @@ QuicHttpServerConnectionFactoryImpl::createQuicServerConnection( REGISTER_FACTORY(QuicHttpClientConnectionFactoryImpl, Http::QuicHttpClientConnectionFactory); REGISTER_FACTORY(QuicHttpServerConnectionFactoryImpl, Http::QuicHttpServerConnectionFactory); +REGISTER_FACTORY(QuicClientConnectionFactoryImpl, Http::QuicClientConnectionFactory); +REGISTER_FACTORY(QuicActiveQuicListenerFactoryImpl, Http::QuicActiveQuicListenerFactory); +REGISTER_FACTORY(QuicServerTransportSocketFactoryImpl, Http::QuicServerTransportSocketFactory); } // namespace Quic } // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/codec_impl.h b/source/extensions/quic_listeners/quiche/codec_impl.h index 8f655523c3cf..fb9a2b58ab13 100644 --- a/source/extensions/quic_listeners/quiche/codec_impl.h +++ b/source/extensions/quic_listeners/quiche/codec_impl.h @@ -6,8 +6,18 @@ #include "common/http/http3/quic_codec_factory.h" #include "common/http/http3/well_known_names.h" +#include "extensions/quic_listeners/quiche/active_quic_listener.h" +#include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" #include "extensions/quic_listeners/quiche/envoy_quic_client_session.h" +#include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" +#include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h" #include "extensions/quic_listeners/quiche/envoy_quic_server_session.h" +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/quic_listeners/quiche/quic_transport_socket_factory.h" +#include "extensions/transport_sockets/tls/ssl_socket.h" + +#include "quiche/quic/core/http/quic_client_push_promise_index.h" +#include "quiche/quic/core/quic_utils.h" namespace Envoy { namespace Quic { @@ -94,8 +104,95 @@ class QuicHttpServerConnectionFactoryImpl : public Http::QuicHttpServerConnectio std::string name() const override { return Http::QuicCodecNames::get().Quiche; } }; +// TODO(#14829) we should avoid creating this per-connection. +struct QuicUpstreamData { + QuicUpstreamData(Event::Dispatcher& dispatcher, Envoy::Ssl::ClientContextConfig& config, + Network::Address::InstanceConstSharedPtr server_addr) + : conn_helper_(dispatcher), alarm_factory_(dispatcher, *conn_helper_.GetClock()), + config_(config), server_id_{config_.serverNameIndication(), + static_cast(server_addr->ip()->port()), false} {} + + EnvoyQuicConnectionHelper conn_helper_; + EnvoyQuicAlarmFactory alarm_factory_; + Envoy::Ssl::ClientContextConfig& config_; + quic::QuicServerId server_id_; + std::unique_ptr crypto_config_; + quic::ParsedQuicVersionVector supported_versions_{quic::CurrentSupportedVersions()}; +}; + +class EnvoyQuicClientSessionWithExtras : public EnvoyQuicClientSession { +public: + using EnvoyQuicClientSession::EnvoyQuicClientSession; + + std::unique_ptr quic_upstream_data_; +}; + +// A factory to create EnvoyQuicClientConnection instance for QUIC +class QuicClientConnectionFactoryImpl : public Http::QuicClientConnectionFactory { +public: + std::unique_ptr + createQuicNetworkConnection(Network::Address::InstanceConstSharedPtr server_addr, + Network::Address::InstanceConstSharedPtr local_addr, + Network::TransportSocketFactory& transport_socket_factory, + Stats::Scope& stats_scope, Event::Dispatcher& dispatcher, + TimeSource& time_source) override { + // TODO(#14829): reject the config if a raw buffer socket is configured. + auto* ssl_socket_factory = + dynamic_cast( + &transport_socket_factory); + ASSERT(ssl_socket_factory != nullptr); + + std::unique_ptr upstream_data = + std::make_unique(dispatcher, *ssl_socket_factory->config(), server_addr); + upstream_data->crypto_config_ = std::make_unique( + std::make_unique(stats_scope, upstream_data->config_, time_source)); + + auto connection = std::make_unique( + quic::QuicUtils::CreateRandomConnectionId(), server_addr, upstream_data->conn_helper_, + upstream_data->alarm_factory_, + quic::ParsedQuicVersionVector{upstream_data->supported_versions_[0]}, local_addr, + dispatcher, nullptr); + auto ret = std::make_unique( + quic_config_, upstream_data->supported_versions_, std::move(connection), + upstream_data->server_id_, upstream_data->crypto_config_.get(), &push_promise_index_, + dispatcher, 0); + ret->Initialize(); + ret->quic_upstream_data_ = std::move(upstream_data); + return ret; + } + + quic::QuicConfig quic_config_; + std::string name() const override { return Http::QuicCodecNames::get().Quiche; } + + quic::QuicClientPushPromiseIndex push_promise_index_; +}; + +class QuicActiveQuicListenerFactoryImpl : public Http::QuicActiveQuicListenerFactory { +public: + Network::ActiveUdpListenerFactoryPtr + createActiveQuicListener(envoy::config::listener::v3::QuicProtocolOptions config, + uint32_t concurrency) override { + return std::make_unique(config, concurrency); + } + std::string name() const override { return Http::QuicCodecNames::get().Quiche; } +}; + +// A factory to create QuicServerTransportSocketFactory instance for QUIC +class QuicServerTransportSocketFactoryImpl : public Http::QuicServerTransportSocketFactory { +public: + Network::TransportSocketFactoryPtr + createQuicServerTransportSocketFactory(Ssl::ServerContextConfigPtr config) override { + return std::make_unique(std::move(config)); + } + + std::string name() const override { return Http::QuicCodecNames::get().Quiche; } +}; + DECLARE_FACTORY(QuicHttpClientConnectionFactoryImpl); DECLARE_FACTORY(QuicHttpServerConnectionFactoryImpl); +DECLARE_FACTORY(QuicClientConnectionFactoryImpl); +DECLARE_FACTORY(QuicActiveQuicListenerFactoryImpl); +DECLARE_FACTORY(QuicServerTransportSocketFactoryImpl); } // namespace Quic } // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.h b/source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.h index 7cd66c14d22c..6d07c2d07d0e 100644 --- a/source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.h +++ b/source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.h @@ -42,8 +42,10 @@ class QuicFilterManagerConnectionImpl : public Network::ConnectionImplBase { void noDelay(bool /*enable*/) override { // No-op. TCP_NODELAY doesn't apply to UDP. } - void readDisable(bool /*disable*/) override { NOT_REACHED_GCOVR_EXCL_LINE; } - void detectEarlyCloseWhenReadDisabled(bool /*value*/) override { NOT_REACHED_GCOVR_EXCL_LINE; } + // TODO(#14829) both readDisable and detectEarlyCloseWhenReadDisabled are used for upstream QUIC + // and needs to be hooked up before it is production-safe. + void readDisable(bool /*disable*/) override {} + void detectEarlyCloseWhenReadDisabled(bool /*value*/) override {} bool readEnabled() const override { return true; } const Network::SocketAddressSetter& addressProvider() const override { return quic_connection_->connectionSocket()->addressProvider(); diff --git a/source/extensions/transport_sockets/tls/ssl_socket.h b/source/extensions/transport_sockets/tls/ssl_socket.h index cb15d82fa935..0a1292ad557b 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.h +++ b/source/extensions/transport_sockets/tls/ssl_socket.h @@ -114,6 +114,8 @@ class ClientSslSocketFactory : public Network::TransportSocketFactory, // Secret::SecretCallbacks void onAddOrUpdateSecret() override; + Envoy::Ssl::ClientContextConfigPtr& config() { return config_; } + private: Envoy::Ssl::ContextManager& manager_; Stats::Scope& stats_scope_; diff --git a/test/common/upstream/test_cluster_manager.h b/test/common/upstream/test_cluster_manager.h index c72023ebac56..a4b4a6048942 100644 --- a/test/common/upstream/test_cluster_manager.h +++ b/test/common/upstream/test_cluster_manager.h @@ -82,7 +82,7 @@ class TestClusterManagerFactory : public ClusterManagerFactory { std::vector&, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsSharedPtr& transport_socket_options, - ClusterConnectivityState& state) override { + TimeSource&, ClusterConnectivityState& state) override { return Http::ConnectionPool::InstancePtr{ allocateConnPool_(host, options, transport_socket_options, state)}; } diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 75632568d3ee..ca9d43bd2ce6 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -3079,24 +3079,27 @@ TEST_F(ClusterInfoImplTest, Http3) { "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions explicit_http_config: http3_protocol_options: {} + common_http_protocol_options: + idle_timeout: 1s )EOF"; const std::string downstream_http3 = R"EOF( typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + use_downstream_protocol_config: + http3_protocol_options: {} common_http_protocol_options: idle_timeout: 1s )EOF"; - { - EXPECT_THROW_WITH_REGEX(makeCluster(yaml + explicit_http3), EnvoyException, - "HTTP3 not yet supported: name.*"); - } - { - EXPECT_THROW_WITH_REGEX(makeCluster(yaml + explicit_http3), EnvoyException, - "HTTP3 not yet supported: name.*"); - } + auto explicit_h3 = makeCluster(yaml + explicit_http3); + EXPECT_EQ(Http::Protocol::Http3, + explicit_h3->info()->upstreamHttpProtocol({Http::Protocol::Http10})[0]); + + auto downstream_h3 = makeCluster(yaml + downstream_http3); + EXPECT_EQ(Http::Protocol::Http3, + downstream_h3->info()->upstreamHttpProtocol({Http::Protocol::Http3})[0]); } // Validate empty singleton for HostsPerLocalityImpl. diff --git a/test/config/utility.cc b/test/config/utility.cc index 13cfc0eaaff9..bba573e5cdf2 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -702,6 +702,8 @@ void ConfigHelper::configureUpstreamTls(bool use_alpn) { tls_context.mutable_common_tls_context()->mutable_validation_context(); validation_context->mutable_trusted_ca()->set_filename( TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); + // The test certs are for *.lyft.com, so make sure SNI matches. + tls_context.set_sni("foo.lyft.com"); cluster->mutable_transport_socket()->set_name("envoy.transport_sockets.tls"); cluster->mutable_transport_socket()->mutable_typed_config()->PackFrom(tls_context); }); diff --git a/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc b/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc index a5ca19bba232..fea82fa204d6 100644 --- a/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc +++ b/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc @@ -61,7 +61,7 @@ class ActiveQuicListenerPeer { return *listener.crypto_config_; } - static bool enabled(ActiveQuicListener& listener) { return listener.enabled_.enabled(); } + static bool enabled(ActiveQuicListener& listener) { return listener.enabled_->enabled(); } }; class ActiveQuicListenerFactoryPeer { diff --git a/test/extensions/quic_listeners/quiche/integration/BUILD b/test/extensions/quic_listeners/quiche/integration/BUILD index 1f5ad0d2e807..45bb6e6a3e1c 100644 --- a/test/extensions/quic_listeners/quiche/integration/BUILD +++ b/test/extensions/quic_listeners/quiche/integration/BUILD @@ -8,6 +8,25 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_test( + name = "quic_protocol_integration_test", + size = "medium", + srcs = [ + "quic_protocol_integration_test.cc", + ], + data = ["//test/config/integration/certs"], + shard_count = 6, + tags = [ + "fails_on_clang_cl", + "fails_on_windows", + "nofips", + ], + deps = [ + "//source/extensions/quic_listeners/quiche:quic_factory_lib", + "//test/integration:protocol_integration_test_lib", + ], +) + envoy_cc_test( name = "quic_http_integration_test", size = "medium", diff --git a/test/extensions/quic_listeners/quiche/integration/quic_protocol_integration_test.cc b/test/extensions/quic_listeners/quiche/integration/quic_protocol_integration_test.cc new file mode 100644 index 000000000000..019f2007f282 --- /dev/null +++ b/test/extensions/quic_listeners/quiche/integration/quic_protocol_integration_test.cc @@ -0,0 +1,15 @@ +#include "test/integration/protocol_integration_test.h" + +namespace Envoy { + +// We do not yet run QUIC downstream tests. +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(DownstreamProtocolIntegrationTest); + +// This will run with HTTP/1 and HTTP/2 downstream, and HTTP/3 upstream. +INSTANTIATE_TEST_SUITE_P(Protocols, ProtocolIntegrationTest, + testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( + {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, + {FakeHttpConnection::Type::HTTP3})), + HttpProtocolIntegrationTest::protocolTestParamsToString); + +} // namespace Envoy diff --git a/test/integration/BUILD b/test/integration/BUILD index a8fc2ddad1a8..a9e54eca07fd 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -680,6 +680,8 @@ envoy_cc_test_library( "//source/common/grpc:common_lib", "//source/common/http/http1:codec_lib", "//source/common/http/http2:codec_lib", + "//source/common/http/http3:quic_codec_factory_lib", + "//source/common/http/http3:well_known_names", "//source/common/network:connection_balancer_lib", "//source/common/network:filter_lib", "//source/common/network:listen_socket_lib", diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 55b5bb99f66a..81983d3003e8 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -123,9 +123,15 @@ Network::TransportSocketFactoryPtr BaseIntegrationTest::createUpstreamTlsContext } auto cfg = std::make_unique( tls_context, factory_context_); - static Stats::Scope* upstream_stats_store = new Stats::IsolatedStoreImpl(); - return std::make_unique( - std::move(cfg), context_manager_, *upstream_stats_store, std::vector{}); + if (upstream_config_.upstream_protocol_ != FakeHttpConnection::Type::HTTP3) { + static Stats::Scope* upstream_stats_store = new Stats::IsolatedStoreImpl(); + return std::make_unique( + std::move(cfg), context_manager_, *upstream_stats_store, std::vector{}); + } else { + return Config::Utility::getAndCheckFactoryByName( + Http::QuicCodecNames::get().Quiche) + .createQuicServerTransportSocketFactory(std::move(cfg)); + } } void BaseIntegrationTest::createUpstreams() { @@ -211,8 +217,7 @@ void BaseIntegrationTest::setUpstreamProtocol(FakeHttpConnection::Type protocol) ConfigHelper::setProtocolOptions( *bootstrap.mutable_static_resources()->mutable_clusters(0), protocol_options); }); - } else { - RELEASE_ASSERT(protocol == FakeHttpConnection::Type::HTTP1, ""); + } else if (upstream_config_.upstream_protocol_ == FakeHttpConnection::Type::HTTP1) { config_helper_.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() >= 1, ""); @@ -221,6 +226,19 @@ void BaseIntegrationTest::setUpstreamProtocol(FakeHttpConnection::Type protocol) ConfigHelper::setProtocolOptions( *bootstrap.mutable_static_resources()->mutable_clusters(0), protocol_options); }); + } else { + RELEASE_ASSERT(protocol == FakeHttpConnection::Type::HTTP3, ""); + setUdpFakeUpstream(true); + upstream_tls_ = true; + config_helper_.configureUpstreamTls(false); + config_helper_.addConfigModifier( + [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() >= 1, ""); + ConfigHelper::HttpProtocolOptions protocol_options; + protocol_options.mutable_explicit_http_config()->mutable_http3_protocol_options(); + ConfigHelper::setProtocolOptions( + *bootstrap.mutable_static_resources()->mutable_clusters(0), protocol_options); + }); } } diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index be4a9a71b9c1..7a5023a4c0a6 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -6,9 +6,11 @@ #include #include "common/buffer/buffer_impl.h" +#include "common/config/utility.h" #include "common/http/header_map_impl.h" #include "common/http/http1/codec_impl.h" #include "common/http/http2/codec_impl.h" +#include "common/http/http3/well_known_names.h" #include "common/network/address_impl.h" #include "common/network/listen_socket_impl.h" #include "common/network/socket_option_factory.h" @@ -30,6 +32,7 @@ using testing::AssertionResult; using testing::AssertionSuccess; namespace Envoy { + FakeStream::FakeStream(FakeHttpConnection& parent, Http::ResponseEncoder& encoder, Event::TestTimeSystem& time_system) : parent_(parent), encoder_(encoder), time_system_(time_system) { @@ -326,7 +329,7 @@ FakeHttpConnection::FakeHttpConnection( codec_ = std::make_unique( shared_connection_.connection(), stats, *this, http1_settings, max_request_headers_kb, max_request_headers_count, headers_with_underscores_action); - } else { + } else if (type == Type::HTTP2) { envoy::config::core::v3::Http2ProtocolOptions http2_options = ::Envoy::Http2::Utility::initializeAndValidateOptions( envoy::config::core::v3::Http2ProtocolOptions()); @@ -336,7 +339,13 @@ FakeHttpConnection::FakeHttpConnection( codec_ = std::make_unique( shared_connection_.connection(), *this, stats, random_, http2_options, max_request_headers_kb, max_request_headers_count, headers_with_underscores_action); - ASSERT(type == Type::HTTP2); + } else { + ASSERT(type == Type::HTTP3); + envoy::config::core::v3::Http3ProtocolOptions http3_options; + codec_ = std::unique_ptr( + Config::Utility::getAndCheckFactoryByName( + Http::QuicCodecNames::get().Quiche) + .createQuicServerConnection(shared_connection_.connection(), *this)); } shared_connection_.connection().addReadFilter( Network::ReadFilterSharedPtr{new ReadFilter(*this)}); @@ -366,20 +375,20 @@ Http::RequestDecoder& FakeHttpConnection::newStream(Http::ResponseEncoder& encod } void FakeHttpConnection::onGoAway(Http::GoAwayErrorCode code) { - ASSERT(type_ == Type::HTTP2); + ASSERT(type_ >= Type::HTTP2); // Usually indicates connection level errors, no operations are needed since // the connection will be closed soon. ENVOY_LOG(info, "FakeHttpConnection receives GOAWAY: ", code); } void FakeHttpConnection::encodeGoAway() { - ASSERT(type_ == Type::HTTP2); + ASSERT(type_ >= Type::HTTP2); shared_connection_.connection().dispatcher().post([this]() { codec_->goAway(); }); } void FakeHttpConnection::encodeProtocolError() { - ASSERT(type_ == Type::HTTP2); + ASSERT(type_ >= Type::HTTP2); Http::Http2::ServerConnectionImpl* codec = dynamic_cast(codec_.get()); @@ -455,11 +464,15 @@ makeUdpListenSocket(const Network::Address::InstanceConstSharedPtr& address) { return socket; } +static Network::SocketPtr +makeListenSocket(const FakeUpstreamConfig& config, + const Network::Address::InstanceConstSharedPtr& address) { + return (config.udp_fake_upstream_ ? makeUdpListenSocket(address) : makeTcpListenSocket(address)); +} + FakeUpstream::FakeUpstream(const Network::Address::InstanceConstSharedPtr& address, const FakeUpstreamConfig& config) - : FakeUpstream(Network::Test::createRawBufferSocketFactory(), - config.udp_fake_upstream_ ? makeUdpListenSocket(address) - : makeTcpListenSocket(address), + : FakeUpstream(Network::Test::createRawBufferSocketFactory(), makeListenSocket(config, address), config) { ENVOY_LOG(info, "starting fake server on socket {}:{}. Address version is {}. UDP={}", address->ip()->addressAsString(), address->ip()->port(), @@ -471,6 +484,7 @@ FakeUpstream::FakeUpstream(uint32_t port, Network::Address::IpVersion version, const FakeUpstreamConfig& config) : FakeUpstream(Network::Test::createRawBufferSocketFactory(), makeTcpListenSocket(port, version), config) { + ASSERT(!config.udp_fake_upstream_); ENVOY_LOG(info, "starting fake server on port {}. Address version is {}", localAddress()->ip()->port(), Network::Test::addressVersionAsString(version)); } @@ -478,10 +492,7 @@ FakeUpstream::FakeUpstream(uint32_t port, Network::Address::IpVersion version, FakeUpstream::FakeUpstream(Network::TransportSocketFactoryPtr&& transport_socket_factory, const Network::Address::InstanceConstSharedPtr& address, const FakeUpstreamConfig& config) - : FakeUpstream(std::move(transport_socket_factory), - config.udp_fake_upstream_ ? makeUdpListenSocket(address) - : makeTcpListenSocket(address), - config) { + : FakeUpstream(std::move(transport_socket_factory), makeListenSocket(config, address), config) { ENVOY_LOG(info, "starting fake server on socket {}:{}. Address version is {}. UDP={}", address->ip()->addressAsString(), address->ip()->port(), Network::Test::addressVersionAsString(address->ip()->version()), @@ -493,6 +504,7 @@ FakeUpstream::FakeUpstream(Network::TransportSocketFactoryPtr&& transport_socket const FakeUpstreamConfig& config) : FakeUpstream(std::move(transport_socket_factory), makeTcpListenSocket(port, version), config) { + ASSERT(!config.udp_fake_upstream_); ENVOY_LOG(info, "starting fake server on port {}. Address version is {}", localAddress()->ip()->port(), Network::Test::addressVersionAsString(version)); } @@ -506,7 +518,7 @@ FakeUpstream::FakeUpstream(Network::TransportSocketFactoryPtr&& transport_socket dispatcher_(api_->allocateDispatcher("fake_upstream")), handler_(new Server::ConnectionHandlerImpl(*dispatcher_, 0)), read_disable_on_new_connection_(true), enable_half_close_(config.enable_half_close_), - listener_(*this), + listener_(*this, http_type_ == FakeHttpConnection::Type::HTTP3), filter_chain_(Network::Test::createEmptyFilterChain(std::move(transport_socket_factory))) { thread_ = api_->threadFactory().createThread([this]() -> void { threadRoutine(); }); server_initialized_.waitReady(); @@ -532,7 +544,20 @@ bool FakeUpstream::createNetworkFilterChain(Network::Connection& connection, connection.readDisable(true); } auto connection_wrapper = std::make_unique(connection); + LinkedList::moveIntoListBack(std::move(connection_wrapper), new_connections_); + + // Normally we don't associate a logical network connection with a FakeHttpConnection until + // waitForHttpConnection is called, but QUIC needs to be set up as packets come in, so we do + // not lazily create for HTTP/3 + // TODO(#14829) handle the case where waitForHttpConnection is called with + // non-default arguments for max request headers and protocol options. + if (http_type_ == FakeHttpConnection::Type::HTTP3) { + quic_connections_.push_back(std::make_unique( + *this, consumeConnection(), http_type_, time_system_, Http::DEFAULT_MAX_REQUEST_HEADERS_KB, + Http::DEFAULT_MAX_HEADERS_COUNT, envoy::config::core::v3::HttpProtocolOptions::ALLOW)); + quic_connections_.back()->initialize(); + } return true; } @@ -562,6 +587,24 @@ AssertionResult FakeUpstream::waitForHttpConnection( headers_with_underscores_action) { { absl::MutexLock lock(&lock_); + + // As noted in createNetworkFilterChain, HTTP3 FakeHttpConnections are not + // lazily created, so HTTP3 needs a different wait path here. + if (http_type_ == FakeHttpConnection::Type::HTTP3) { + if (quic_connections_.empty() && + !waitForWithDispatcherRun( + time_system_, lock_, + [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { return !quic_connections_.empty(); }, + client_dispatcher, timeout)) { + return AssertionFailure() << "Timed out waiting for new quic connection."; + } + if (!quic_connections_.empty()) { + connection = std::move(quic_connections_.front()); + quic_connections_.pop_front(); + return AssertionSuccess(); + } + } + if (!waitForWithDispatcherRun( time_system_, lock_, [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { return !new_connections_.empty(); }, @@ -678,6 +721,7 @@ testing::AssertionResult FakeUpstream::waitForUdpDatagram(Network::UdpRecvData& } void FakeUpstream::onRecvDatagram(Network::UdpRecvData& data) { + std::cerr << "Got datagrams. Need to dispatch\n"; absl::MutexLock lock(&lock_); received_datagrams_.emplace_back(std::move(data)); } diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 69ff23013a2a..ac29d5c8dd11 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -21,10 +21,13 @@ #include "common/common/linked_object.h" #include "common/common/lock_guard.h" #include "common/common/thread.h" +#include "common/config/utility.h" #include "common/grpc/codec.h" #include "common/grpc/common.h" #include "common/http/http1/codec_impl.h" #include "common/http/http2/codec_impl.h" +#include "common/http/http3/quic_codec_factory.h" +#include "common/http/http3/well_known_names.h" #include "common/network/connection_balancer_impl.h" #include "common/network/filter_impl.h" #include "common/network/listen_socket_impl.h" @@ -401,7 +404,7 @@ class FakeConnectionBase : public Logger::Loggable { */ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeConnectionBase { public: - enum class Type { HTTP1, HTTP2 }; + enum class Type { HTTP1, HTTP2, HTTP3 }; FakeHttpConnection(FakeUpstream& fake_upstream, SharedConnectionWrapper& shared_connection, Type type, Event::TestTimeSystem& time_system, uint32_t max_request_headers_kb, @@ -416,13 +419,13 @@ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeCo // Http::ServerConnectionCallbacks Http::RequestDecoder& newStream(Http::ResponseEncoder& response_encoder, bool) override; - // Should only be called for HTTP2 + // Should only be called for HTTP2 or above void onGoAway(Http::GoAwayErrorCode code) override; - // Should only be called for HTTP2, sends a GOAWAY frame with NO_ERROR. + // Should only be called for HTTP2 or above, sends a GOAWAY frame with NO_ERROR. void encodeGoAway(); - // Should only be called for HTTP2, sends a GOAWAY frame with ENHANCE_YOUR_CALM. + // Should only be called for HTTP2 or above, sends a GOAWAY frame with ENHANCE_YOUR_CALM. void encodeProtocolError(); private: @@ -678,11 +681,20 @@ class FakeUpstream : Logger::Loggable, class FakeListener : public Network::ListenerConfig { public: - FakeListener(FakeUpstream& parent) + FakeListener(FakeUpstream& parent, bool is_quic = false) : parent_(parent), name_("fake_upstream"), - udp_listener_factory_(std::make_unique(1)), udp_writer_factory_(std::make_unique()), - udp_listener_worker_router_(1), init_manager_(nullptr) {} + udp_listener_worker_router_(1), init_manager_(nullptr) { + if (is_quic) { + envoy::config::listener::v3::QuicProtocolOptions config; + udp_listener_factory_ = + Config::Utility::getAndCheckFactoryByName( + Http::QuicCodecNames::get().Quiche) + .createActiveQuicListener(config, 1); + } else { + udp_listener_factory_ = std::make_unique(1); + } + } private: // Network::ListenerConfig @@ -727,7 +739,7 @@ class FakeUpstream : Logger::Loggable, FakeUpstream& parent_; const std::string name_; Network::NopConnectionBalancerImpl connection_balancer_; - const Network::ActiveUdpListenerFactoryPtr udp_listener_factory_; + Network::ActiveUdpListenerFactoryPtr udp_listener_factory_; const Network::UdpPacketWriterFactoryPtr udp_writer_factory_; Network::UdpListenerWorkerRouterImpl udp_listener_worker_router_; BasicResourceLimitImpl connection_resource_; @@ -754,6 +766,8 @@ class FakeUpstream : Logger::Loggable, Event::DispatcherPtr dispatcher_; Network::ConnectionHandlerPtr handler_; std::list new_connections_ ABSL_GUARDED_BY(lock_); + std::list quic_connections_ ABSL_GUARDED_BY(lock_); + // When a QueuedConnectionWrapper is popped from new_connections_, ownership is transferred to // consumed_connections_. This allows later the Connection destruction (when the FakeUpstream is // deleted) on the same thread that allocated the connection. diff --git a/test/integration/http_protocol_integration.cc b/test/integration/http_protocol_integration.cc index b22421587e87..9420f33f1fb6 100644 --- a/test/integration/http_protocol_integration.cc +++ b/test/integration/http_protocol_integration.cc @@ -18,14 +18,25 @@ std::vector HttpProtocolIntegrationTest::getProtocolTest return ret; } +absl::string_view upstreamToString(FakeHttpConnection::Type type) { + switch (type) { + case FakeHttpConnection::Type::HTTP1: + return "HttpUpstream"; + case FakeHttpConnection::Type::HTTP2: + return "Http2Upstream"; + case FakeHttpConnection::Type::HTTP3: + return "Http3Upstream"; + } + return "UnknownUpstream"; +} + std::string HttpProtocolIntegrationTest::protocolTestParamsToString( const ::testing::TestParamInfo& params) { - return absl::StrCat( - (params.param.version == Network::Address::IpVersion::v4 ? "IPv4_" : "IPv6_"), - (params.param.downstream_protocol == Http::CodecClient::Type::HTTP2 ? "Http2Downstream_" - : "HttpDownstream_"), - (params.param.upstream_protocol == FakeHttpConnection::Type::HTTP2 ? "Http2Upstream" - : "HttpUpstream")); + return absl::StrCat((params.param.version == Network::Address::IpVersion::v4 ? "IPv4_" : "IPv6_"), + (params.param.downstream_protocol == Http::CodecClient::Type::HTTP2 + ? "Http2Downstream_" + : "HttpDownstream_"), + upstreamToString(params.param.upstream_protocol)); } } // namespace Envoy diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 8ebd3ce22c33..ef2edf4b26a3 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -53,6 +53,12 @@ void setDoNotValidateRouteConfig( route_config->mutable_validate_clusters()->set_value(false); }; +// TODO(#14829) categorize or fix all failures. +#define EXCLUDE_UPSTREAM_HTTP3 \ + if (upstreamProtocol() == FakeHttpConnection::Type::HTTP3) { \ + return; \ + } + TEST_P(ProtocolIntegrationTest, TrailerSupportHttp1) { config_helper_.addConfigModifier(setEnableDownstreamTrailersHttp1()); config_helper_.addConfigModifier(setEnableUpstreamTrailersHttp1()); @@ -202,6 +208,7 @@ TEST_P(ProtocolIntegrationTest, AddBodyToRequestAndWaitForIt) { } TEST_P(ProtocolIntegrationTest, AddBodyToResponseAndWaitForIt) { + EXCLUDE_UPSTREAM_HTTP3; // filters are prepended, so add them in reverse order config_helper_.addFilter(R"EOF( name: add-body-filter @@ -224,6 +231,7 @@ TEST_P(ProtocolIntegrationTest, AddBodyToResponseAndWaitForIt) { } TEST_P(ProtocolIntegrationTest, ContinueHeadersOnlyInjectBodyFilter) { + EXCLUDE_UPSTREAM_HTTP3; config_helper_.addFilter(R"EOF( name: continue-headers-only-inject-body-filter typed_config: @@ -292,7 +300,7 @@ name: add-trailers-filter upstream_request_->encodeData(128, true); response->waitForEndStream(); - if (upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { + if (upstreamProtocol() >= FakeHttpConnection::Type::HTTP2) { EXPECT_EQ("decode", upstream_request_->trailers() ->get(Http::LowerCaseString("grpc-message"))[0] ->value() @@ -325,7 +333,7 @@ TEST_P(ProtocolIntegrationTest, ResponseWithHostHeader) { } // Tests missing headers needed for H/1 codec first line. -TEST_P(ProtocolIntegrationTest, DownstreamRequestWithFaultyFilter) { +TEST_P(DownstreamProtocolIntegrationTest, DownstreamRequestWithFaultyFilter) { useAccessLog("%RESPONSE_CODE_DETAILS%"); config_helper_.addFilter("{ name: invalid-header-filter, typed_config: { \"@type\": " "type.googleapis.com/google.protobuf.Empty } }"); @@ -357,7 +365,7 @@ TEST_P(ProtocolIntegrationTest, DownstreamRequestWithFaultyFilter) { EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("missing_required_header")); } -TEST_P(ProtocolIntegrationTest, FaultyFilterWithConnect) { +TEST_P(DownstreamProtocolIntegrationTest, FaultyFilterWithConnect) { // Faulty filter that removed host in a CONNECT request. config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& @@ -413,6 +421,7 @@ TEST_P(DownstreamProtocolIntegrationTest, MissingHeadersLocalReply) { // Regression test for https://github.com/envoyproxy/envoy/issues/10270 TEST_P(ProtocolIntegrationTest, LongHeaderValueWithSpaces) { + EXCLUDE_UPSTREAM_HTTP3; // Header with at least 20kb of spaces surrounded by non-whitespace characters to ensure that // dispatching is split across 2 dispatch calls. This threshold comes from Envoy preferring 16KB // reads, which the buffer rounds up to about 20KB when allocating slices in @@ -450,6 +459,7 @@ TEST_P(ProtocolIntegrationTest, LongHeaderValueWithSpaces) { } TEST_P(ProtocolIntegrationTest, Retry) { + EXCLUDE_UPSTREAM_HTTP3; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); auto response = codec_client_->makeRequestWithBody( @@ -494,6 +504,7 @@ TEST_P(ProtocolIntegrationTest, Retry) { } TEST_P(ProtocolIntegrationTest, RetryStreaming) { + EXCLUDE_UPSTREAM_HTTP3; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); auto encoder_decoder = @@ -551,6 +562,7 @@ TEST_P(ProtocolIntegrationTest, RetryStreaming) { // sure that Envoy cleans up stream state correctly when doing a retry with // complete response but incomplete request. TEST_P(ProtocolIntegrationTest, RetryStreamingReset) { + EXCLUDE_UPSTREAM_HTTP3; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); auto encoder_decoder = @@ -992,6 +1004,7 @@ TEST_P(ProtocolIntegrationTest, HittingEncoderFilterLimit) { // connection error is not detected under these circumstances. #if !defined(__APPLE__) TEST_P(ProtocolIntegrationTest, 100ContinueAndClose) { + EXCLUDE_UPSTREAM_HTTP3; // TODO(#14829) 503 testEnvoyHandling100Continue(false, "", true); } #endif @@ -1020,7 +1033,10 @@ TEST_P(ProtocolIntegrationTest, EnvoyProxyingLateMultiple1xx) { TEST_P(ProtocolIntegrationTest, TwoRequests) { testTwoRequests(); } -TEST_P(ProtocolIntegrationTest, TwoRequestsWithForcedBackup) { testTwoRequests(true); } +TEST_P(ProtocolIntegrationTest, TwoRequestsWithForcedBackup) { + EXCLUDE_UPSTREAM_HTTP3; + testTwoRequests(true); +} TEST_P(ProtocolIntegrationTest, BasicMaxStreamDuration) { testMaxStreamDuration(); } @@ -1174,12 +1190,13 @@ TEST_P(ProtocolIntegrationTest, 304WithBody) { // can differentiate between 304-with-advertised-but-absent-body and // 304-with-body, is there a protocol error on the active stream. if (downstream_protocol_ == Http::CodecClient::Type::HTTP2 && - upstreamProtocol() == FakeHttpConnection::Type::HTTP2) { + upstreamProtocol() >= FakeHttpConnection::Type::HTTP2) { response->waitForReset(); } } TEST_P(ProtocolIntegrationTest, OverflowingResponseCode) { + EXCLUDE_UPSTREAM_HTTP3; // Protocol-specific framing. initialize(); FakeRawConnectionPtr fake_upstream_connection; @@ -1215,6 +1232,7 @@ TEST_P(ProtocolIntegrationTest, OverflowingResponseCode) { } TEST_P(ProtocolIntegrationTest, MissingStatus) { + EXCLUDE_UPSTREAM_HTTP3; // Protocol-specific framing. initialize(); FakeRawConnectionPtr fake_upstream_connection; @@ -1601,6 +1619,7 @@ TEST_P(ProtocolIntegrationTest, LargeRequestMethod) { // There will be no upstream connections for HTTP/1 downstream, we need to // test the full mesh regardless. testing_upstream_intentionally_ = true; + EXCLUDE_UPSTREAM_HTTP3; // TODO(#14829) Rejected with QUIC_STREAM_EXCESSIVE_LOAD const std::string long_method = std::string(48 * 1024, 'a'); const Http::TestRequestHeaderMapImpl request_headers{{":method", long_method}, {":path", "/test/long/url"}, @@ -1627,7 +1646,7 @@ TEST_P(ProtocolIntegrationTest, LargeRequestMethod) { EXPECT_TRUE(response->complete()); EXPECT_EQ("400", response->headers().getStatusValue()); } else { - ASSERT(upstreamProtocol() == FakeHttpConnection::Type::HTTP2); + ASSERT(upstreamProtocol() >= FakeHttpConnection::Type::HTTP2); auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); EXPECT_TRUE(response->complete()); @@ -1868,6 +1887,7 @@ name: encode-headers-return-stop-all-filter // Per https://github.com/envoyproxy/envoy/issues/7488 make sure we don't // combine set-cookie headers TEST_P(ProtocolIntegrationTest, MultipleSetCookies) { + EXCLUDE_UPSTREAM_HTTP3; // Multiple cookies not yet supported. initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); diff --git a/test/mocks/upstream/cluster_manager_factory.h b/test/mocks/upstream/cluster_manager_factory.h index 0de2928c57b1..91cad4034acb 100644 --- a/test/mocks/upstream/cluster_manager_factory.h +++ b/test/mocks/upstream/cluster_manager_factory.h @@ -25,7 +25,7 @@ class MockClusterManagerFactory : public ClusterManagerFactory { std::vector& protocol, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsSharedPtr& transport_socket_options, - ClusterConnectivityState& state)); + TimeSource& source, ClusterConnectivityState& state)); MOCK_METHOD(Tcp::ConnectionPool::InstancePtr, allocateTcpConnPool, (Event::Dispatcher & dispatcher, HostConstSharedPtr host, ResourcePriority priority, From 546ec7339b5086599654a01fd046eacfcf45e2dc Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Tue, 16 Feb 2021 14:28:33 -0500 Subject: [PATCH 2/8] fixing factories, adding cerr logging for v6 Ci failures Signed-off-by: Alyssa Wilk --- source/common/http/http3/quic_codec_factory.h | 23 ----------------- .../quic_listeners/quiche/codec_impl.cc | 2 -- .../quic_listeners/quiche/codec_impl.h | 25 ------------------- .../quic_listeners/quiche/integration/BUILD | 2 ++ test/integration/BUILD | 1 + test/integration/base_integration_test.cc | 16 ++++++++---- test/integration/fake_upstream.h | 8 +++--- 7 files changed, 18 insertions(+), 59 deletions(-) diff --git a/source/common/http/http3/quic_codec_factory.h b/source/common/http/http3/quic_codec_factory.h index 030362d28387..5377dee99743 100644 --- a/source/common/http/http3/quic_codec_factory.h +++ b/source/common/http/http3/quic_codec_factory.h @@ -48,28 +48,5 @@ class QuicClientConnectionFactory : public Config::UntypedFactory { std::string category() const override { return "envoy.quic_connection"; } }; -// A factory to create ActiveQuicListenerFactory instance for QUIC -class QuicActiveQuicListenerFactory : public Config::UntypedFactory { -public: - ~QuicActiveQuicListenerFactory() override = default; - - virtual Network::ActiveUdpListenerFactoryPtr - createActiveQuicListener(envoy::config::listener::v3::QuicProtocolOptions, - uint32_t concurrency) PURE; - - std::string category() const override { return "envoy.quic_listener"; } -}; - -// A factory to create QuicServerTransportSocketFactory instance for QUIC -class QuicServerTransportSocketFactory : public Config::UntypedFactory { -public: - ~QuicServerTransportSocketFactory() override = default; - - virtual Network::TransportSocketFactoryPtr - createQuicServerTransportSocketFactory(Ssl::ServerContextConfigPtr config) PURE; - - std::string category() const override { return "envoy.quic_server_transport_factory"; } -}; - } // namespace Http } // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/codec_impl.cc b/source/extensions/quic_listeners/quiche/codec_impl.cc index 75aede9d3dea..730008129e59 100644 --- a/source/extensions/quic_listeners/quiche/codec_impl.cc +++ b/source/extensions/quic_listeners/quiche/codec_impl.cc @@ -114,8 +114,6 @@ QuicHttpServerConnectionFactoryImpl::createQuicServerConnection( REGISTER_FACTORY(QuicHttpClientConnectionFactoryImpl, Http::QuicHttpClientConnectionFactory); REGISTER_FACTORY(QuicHttpServerConnectionFactoryImpl, Http::QuicHttpServerConnectionFactory); REGISTER_FACTORY(QuicClientConnectionFactoryImpl, Http::QuicClientConnectionFactory); -REGISTER_FACTORY(QuicActiveQuicListenerFactoryImpl, Http::QuicActiveQuicListenerFactory); -REGISTER_FACTORY(QuicServerTransportSocketFactoryImpl, Http::QuicServerTransportSocketFactory); } // namespace Quic } // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/codec_impl.h b/source/extensions/quic_listeners/quiche/codec_impl.h index fb9a2b58ab13..71909c7fc05c 100644 --- a/source/extensions/quic_listeners/quiche/codec_impl.h +++ b/source/extensions/quic_listeners/quiche/codec_impl.h @@ -6,14 +6,12 @@ #include "common/http/http3/quic_codec_factory.h" #include "common/http/http3/well_known_names.h" -#include "extensions/quic_listeners/quiche/active_quic_listener.h" #include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" #include "extensions/quic_listeners/quiche/envoy_quic_client_session.h" #include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" #include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h" #include "extensions/quic_listeners/quiche/envoy_quic_server_session.h" #include "extensions/quic_listeners/quiche/envoy_quic_utils.h" -#include "extensions/quic_listeners/quiche/quic_transport_socket_factory.h" #include "extensions/transport_sockets/tls/ssl_socket.h" #include "quiche/quic/core/http/quic_client_push_promise_index.h" @@ -167,32 +165,9 @@ class QuicClientConnectionFactoryImpl : public Http::QuicClientConnectionFactory quic::QuicClientPushPromiseIndex push_promise_index_; }; -class QuicActiveQuicListenerFactoryImpl : public Http::QuicActiveQuicListenerFactory { -public: - Network::ActiveUdpListenerFactoryPtr - createActiveQuicListener(envoy::config::listener::v3::QuicProtocolOptions config, - uint32_t concurrency) override { - return std::make_unique(config, concurrency); - } - std::string name() const override { return Http::QuicCodecNames::get().Quiche; } -}; - -// A factory to create QuicServerTransportSocketFactory instance for QUIC -class QuicServerTransportSocketFactoryImpl : public Http::QuicServerTransportSocketFactory { -public: - Network::TransportSocketFactoryPtr - createQuicServerTransportSocketFactory(Ssl::ServerContextConfigPtr config) override { - return std::make_unique(std::move(config)); - } - - std::string name() const override { return Http::QuicCodecNames::get().Quiche; } -}; - DECLARE_FACTORY(QuicHttpClientConnectionFactoryImpl); DECLARE_FACTORY(QuicHttpServerConnectionFactoryImpl); DECLARE_FACTORY(QuicClientConnectionFactoryImpl); -DECLARE_FACTORY(QuicActiveQuicListenerFactoryImpl); -DECLARE_FACTORY(QuicServerTransportSocketFactoryImpl); } // namespace Quic } // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/integration/BUILD b/test/extensions/quic_listeners/quiche/integration/BUILD index 45bb6e6a3e1c..dcd761caede9 100644 --- a/test/extensions/quic_listeners/quiche/integration/BUILD +++ b/test/extensions/quic_listeners/quiche/integration/BUILD @@ -22,7 +22,9 @@ envoy_cc_test( "nofips", ], deps = [ + "//source/extensions/quic_listeners/quiche:active_quic_listener_lib", "//source/extensions/quic_listeners/quiche:quic_factory_lib", + "//source/extensions/quic_listeners/quiche:quic_transport_socket_factory_lib", "//test/integration:protocol_integration_test_lib", ], ) diff --git a/test/integration/BUILD b/test/integration/BUILD index a9e54eca07fd..292fa7a9b497 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -730,6 +730,7 @@ envoy_cc_test_library( "@envoy_api//envoy/api/v2:pkg_cc_proto", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/transport_sockets/quic/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", ], ) diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 81983d3003e8..4015fd135a7a 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -12,6 +12,7 @@ #include "envoy/buffer/buffer.h" #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/config/endpoint/v3/endpoint_components.pb.h" +#include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.h" #include "envoy/extensions/transport_sockets/tls/v3/cert.pb.h" #include "common/common/assert.h" @@ -121,16 +122,21 @@ Network::TransportSocketFactoryPtr BaseIntegrationTest::createUpstreamTlsContext } else if (upstream_config_.upstream_protocol_ == FakeHttpConnection::Type::HTTP1) { tls_context.mutable_common_tls_context()->add_alpn_protocols("http/1.1"); } - auto cfg = std::make_unique( - tls_context, factory_context_); if (upstream_config_.upstream_protocol_ != FakeHttpConnection::Type::HTTP3) { + auto cfg = std::make_unique( + tls_context, factory_context_); static Stats::Scope* upstream_stats_store = new Stats::IsolatedStoreImpl(); return std::make_unique( std::move(cfg), context_manager_, *upstream_stats_store, std::vector{}); } else { - return Config::Utility::getAndCheckFactoryByName( - Http::QuicCodecNames::get().Quiche) - .createQuicServerTransportSocketFactory(std::move(cfg)); + envoy::extensions::transport_sockets::quic::v3::QuicDownstreamTransport quic_config; + quic_config.mutable_downstream_tls_context()->MergeFrom(tls_context); + + std::vector server_names; + auto& config_factory = Config::Utility::getAndCheckFactoryByName< + Server::Configuration::DownstreamTransportSocketConfigFactory>( + "envoy.transport_sockets.quic"); + return config_factory.createTransportSocketFactory(quic_config, factory_context_, server_names); } } diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index ac29d5c8dd11..989d16e9f6ac 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -687,10 +687,10 @@ class FakeUpstream : Logger::Loggable, udp_listener_worker_router_(1), init_manager_(nullptr) { if (is_quic) { envoy::config::listener::v3::QuicProtocolOptions config; - udp_listener_factory_ = - Config::Utility::getAndCheckFactoryByName( - Http::QuicCodecNames::get().Quiche) - .createActiveQuicListener(config, 1); + auto& config_factory = + Config::Utility::getAndCheckFactoryByName( + "quiche_quic_listener"); + udp_listener_factory_ = config_factory.createActiveUdpListenerFactory(config, 1); } else { udp_listener_factory_ = std::make_unique(1); } From 22bc9007be124aa64d78a067664440ce8e78e9c8 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Tue, 16 Feb 2021 15:05:56 -0500 Subject: [PATCH 3/8] missing files Signed-off-by: Alyssa Wilk --- source/common/http/http3/conn_pool.cc | 2 ++ source/common/network/utility.cc | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/source/common/http/http3/conn_pool.cc b/source/common/http/http3/conn_pool.cc index f4eaf22e2bb1..c498b7159887 100644 --- a/source/common/http/http3/conn_pool.cc +++ b/source/common/http/http3/conn_pool.cc @@ -34,6 +34,8 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ auto source_address = data.host_description_->cluster().sourceAddress(); if (!source_address.get()) { source_address = Network::Utility::getLocalAddress(host_address->ip()->version()); + // TODO(alyssawilk) debug log: fix. + std::cerr << "Local address resolved to " << source_address->asString(); } Network::TransportSocketFactory& transport_socket_factory = data.host_description_->transportSocketFactory(); diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index faee96e38521..e61739038362 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -538,6 +538,12 @@ Api::IoCallUint64Result Utility::writeToSocket(IoHandle& handle, Buffer::RawSlic if (send_result.ok()) { ENVOY_LOG_MISC(trace, "sendmsg bytes {}", send_result.rc_); } else { + // TODO(alyssawilk) REMOVE BEFORE SUBMITTING. + std::cerr << "Failed sending. Peer address " << peer_address.asString(); + if (local_ip) { + std::cerr << " local ip " << local_ip->addressAsString(); + } + std::cerr << std::endl; ENVOY_LOG_MISC(debug, "sendmsg failed with error code {}: {}", static_cast(send_result.err_->getErrorCode()), send_result.err_->getErrorDetails()); From 4f35951b5748a2986e02b09683b7de7b2779b770 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Tue, 16 Feb 2021 15:37:36 -0500 Subject: [PATCH 4/8] format Signed-off-by: Alyssa Wilk --- source/common/network/utility.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index e61739038362..4c4573e1e1fe 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -539,7 +539,7 @@ Api::IoCallUint64Result Utility::writeToSocket(IoHandle& handle, Buffer::RawSlic ENVOY_LOG_MISC(trace, "sendmsg bytes {}", send_result.rc_); } else { // TODO(alyssawilk) REMOVE BEFORE SUBMITTING. - std::cerr << "Failed sending. Peer address " << peer_address.asString(); + std::cerr << "Failed sending. Peer address " << peer_address.asString(); if (local_ip) { std::cerr << " local ip " << local_ip->addressAsString(); } From d6fccfa7d11ac5ea1fd2bb662b99cc9172a4ea55 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Wed, 17 Feb 2021 11:48:04 -0500 Subject: [PATCH 5/8] more factory cleanup, hopeful v6 fix Signed-off-by: Alyssa Wilk --- source/common/http/http3/BUILD | 10 +++ source/common/http/http3/conn_pool.cc | 4 +- .../http3/quic_client_connection_factory.h | 29 ++++++++ source/common/http/http3/quic_codec_factory.h | 17 ----- source/common/network/utility.cc | 6 -- source/extensions/quic_listeners/quiche/BUILD | 24 ++++++- .../quiche/client_connection_factory_impl.cc | 40 +++++++++++ .../quiche/client_connection_factory_impl.h | 61 ++++++++++++++++ .../quic_listeners/quiche/codec_impl.cc | 1 - .../quic_listeners/quiche/codec_impl.h | 72 ------------------- .../quic_listeners/quiche/integration/BUILD | 1 + test/integration/BUILD | 1 + test/integration/base_integration_test.cc | 11 +++ test/integration/fake_upstream.h | 1 + 14 files changed, 176 insertions(+), 102 deletions(-) create mode 100644 source/common/http/http3/quic_client_connection_factory.h create mode 100644 source/extensions/quic_listeners/quiche/client_connection_factory_impl.cc create mode 100644 source/extensions/quic_listeners/quiche/client_connection_factory_impl.h diff --git a/source/common/http/http3/BUILD b/source/common/http/http3/BUILD index 6451b69c3f97..857cbe1a3648 100644 --- a/source/common/http/http3/BUILD +++ b/source/common/http/http3/BUILD @@ -13,6 +13,7 @@ envoy_cc_library( srcs = ["conn_pool.cc"], hdrs = ["conn_pool.h"], deps = [ + ":quic_client_connection_factory_lib", "//include/envoy/event:dispatcher_interface", "//include/envoy/upstream:upstream_interface", "//source/common/http:codec_client_lib", @@ -28,6 +29,15 @@ envoy_cc_library( "//include/envoy/config:typed_config_interface", "//include/envoy/http:codec_interface", "//include/envoy/network:connection_interface", + ], +) + +envoy_cc_library( + name = "quic_client_connection_factory_lib", + hdrs = ["quic_client_connection_factory.h"], + deps = [ + "//include/envoy/config:typed_config_interface", + "//include/envoy/network:connection_interface", "//include/envoy/ssl:context_config_interface", "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", ], diff --git a/source/common/http/http3/conn_pool.cc b/source/common/http/http3/conn_pool.cc index c498b7159887..9165d971cb8c 100644 --- a/source/common/http/http3/conn_pool.cc +++ b/source/common/http/http3/conn_pool.cc @@ -6,7 +6,7 @@ #include "envoy/upstream/upstream.h" #include "common/config/utility.h" -#include "common/http/http3/quic_codec_factory.h" +#include "common/http/http3/quic_client_connection_factory.h" #include "common/http/http3/well_known_names.h" #include "common/http/utility.h" #include "common/network/address_impl.h" @@ -34,8 +34,6 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ auto source_address = data.host_description_->cluster().sourceAddress(); if (!source_address.get()) { source_address = Network::Utility::getLocalAddress(host_address->ip()->version()); - // TODO(alyssawilk) debug log: fix. - std::cerr << "Local address resolved to " << source_address->asString(); } Network::TransportSocketFactory& transport_socket_factory = data.host_description_->transportSocketFactory(); diff --git a/source/common/http/http3/quic_client_connection_factory.h b/source/common/http/http3/quic_client_connection_factory.h new file mode 100644 index 000000000000..1e839b9fb020 --- /dev/null +++ b/source/common/http/http3/quic_client_connection_factory.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include "envoy/config/listener/v3/quic_config.pb.h" +#include "envoy/config/typed_config.h" +#include "envoy/network/connection.h" +#include "envoy/ssl/context_config.h" + +namespace Envoy { +namespace Http { + +// A factory to create EnvoyQuicClientConnection instance for QUIC +class QuicClientConnectionFactory : public Config::UntypedFactory { +public: + ~QuicClientConnectionFactory() override = default; + + virtual std::unique_ptr + createQuicNetworkConnection(Network::Address::InstanceConstSharedPtr server_addr, + Network::Address::InstanceConstSharedPtr local_addr, + Network::TransportSocketFactory& transport_socket_factory, + Stats::Scope& stats_scope, Event::Dispatcher& dispatcher, + TimeSource& time_source) PURE; + + std::string category() const override { return "envoy.quic_connection"; } +}; + +} // namespace Http +} // namespace Envoy diff --git a/source/common/http/http3/quic_codec_factory.h b/source/common/http/http3/quic_codec_factory.h index 5377dee99743..0b4a72404200 100644 --- a/source/common/http/http3/quic_codec_factory.h +++ b/source/common/http/http3/quic_codec_factory.h @@ -2,11 +2,9 @@ #include -#include "envoy/config/listener/v3/quic_config.pb.h" #include "envoy/config/typed_config.h" #include "envoy/http/codec.h" #include "envoy/network/connection.h" -#include "envoy/ssl/context_config.h" namespace Envoy { namespace Http { @@ -33,20 +31,5 @@ class QuicHttpClientConnectionFactory : public Config::UntypedFactory { std::string category() const override { return "envoy.quic_server_codec"; } }; -// A factory to create EnvoyQuicClientConnection instance for QUIC -class QuicClientConnectionFactory : public Config::UntypedFactory { -public: - ~QuicClientConnectionFactory() override = default; - - virtual std::unique_ptr - createQuicNetworkConnection(Network::Address::InstanceConstSharedPtr server_addr, - Network::Address::InstanceConstSharedPtr local_addr, - Network::TransportSocketFactory& transport_socket_factory, - Stats::Scope& stats_scope, Event::Dispatcher& dispatcher, - TimeSource& time_source) PURE; - - std::string category() const override { return "envoy.quic_connection"; } -}; - } // namespace Http } // namespace Envoy diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index 4c4573e1e1fe..faee96e38521 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -538,12 +538,6 @@ Api::IoCallUint64Result Utility::writeToSocket(IoHandle& handle, Buffer::RawSlic if (send_result.ok()) { ENVOY_LOG_MISC(trace, "sendmsg bytes {}", send_result.rc_); } else { - // TODO(alyssawilk) REMOVE BEFORE SUBMITTING. - std::cerr << "Failed sending. Peer address " << peer_address.asString(); - if (local_ip) { - std::cerr << " local ip " << local_ip->addressAsString(); - } - std::cerr << std::endl; ENVOY_LOG_MISC(debug, "sendmsg failed with error code {}: {}", static_cast(send_result.err_->getErrorCode()), send_result.err_->getErrorDetails()); diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 75a5f9a08656..3b29e07fa0a3 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -136,9 +136,9 @@ envoy_cc_library( ) envoy_cc_library( - name = "codec_lib", - srcs = ["codec_impl.cc"], - hdrs = ["codec_impl.h"], + name = "client_connection_factory_lib", + srcs = ["client_connection_factory_impl.cc"], + hdrs = ["client_connection_factory_impl.h"], tags = ["nofips"], deps = [ ":active_quic_listener_lib", @@ -150,6 +150,7 @@ envoy_cc_library( ":envoy_quic_utils_lib", "//include/envoy/http:codec_interface", "//include/envoy/registry", + "//source/common/http/http3:quic_client_connection_factory_lib", "//source/common/http/http3:quic_codec_factory_lib", "//source/common/http/http3:well_known_names", "//source/extensions/transport_sockets/tls:ssl_socket_lib", @@ -157,6 +158,23 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "codec_lib", + srcs = ["codec_impl.cc"], + hdrs = ["codec_impl.h"], + tags = ["nofips"], + deps = [ + ":envoy_quic_client_session_lib", + ":envoy_quic_server_session_lib", + ":envoy_quic_utils_lib", + "//include/envoy/http:codec_interface", + "//include/envoy/registry", + "//source/common/http/http3:quic_codec_factory_lib", + "//source/common/http/http3:well_known_names", + "@com_googlesource_quiche//:quic_core_http_spdy_session_lib", + ], +) + envoy_cc_library( name = "quic_filter_manager_connection_lib", srcs = ["quic_filter_manager_connection_impl.cc"], diff --git a/source/extensions/quic_listeners/quiche/client_connection_factory_impl.cc b/source/extensions/quic_listeners/quiche/client_connection_factory_impl.cc new file mode 100644 index 000000000000..75d0fa6e0059 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/client_connection_factory_impl.cc @@ -0,0 +1,40 @@ +#include "extensions/quic_listeners/quiche/client_connection_factory_impl.h" + +namespace Envoy { +namespace Quic { + +std::unique_ptr +QuicClientConnectionFactoryImpl::createQuicNetworkConnection( + Network::Address::InstanceConstSharedPtr server_addr, + Network::Address::InstanceConstSharedPtr local_addr, + Network::TransportSocketFactory& transport_socket_factory, Stats::Scope& stats_scope, + Event::Dispatcher& dispatcher, TimeSource& time_source) { + // TODO(#14829): reject the config if a raw buffer socket is configured. + auto* ssl_socket_factory = + dynamic_cast( + &transport_socket_factory); + ASSERT(ssl_socket_factory != nullptr); + + std::unique_ptr upstream_data = + std::make_unique(dispatcher, *ssl_socket_factory->config(), server_addr); + upstream_data->crypto_config_ = std::make_unique( + std::make_unique(stats_scope, upstream_data->config_, time_source)); + + auto connection = std::make_unique( + quic::QuicUtils::CreateRandomConnectionId(), server_addr, upstream_data->conn_helper_, + upstream_data->alarm_factory_, + quic::ParsedQuicVersionVector{upstream_data->supported_versions_[0]}, local_addr, dispatcher, + nullptr); + auto ret = std::make_unique( + quic_config_, upstream_data->supported_versions_, std::move(connection), + upstream_data->server_id_, upstream_data->crypto_config_.get(), &push_promise_index_, + dispatcher, 0); + ret->Initialize(); + ret->quic_upstream_data_ = std::move(upstream_data); + return ret; +} + +REGISTER_FACTORY(QuicClientConnectionFactoryImpl, Http::QuicClientConnectionFactory); + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/client_connection_factory_impl.h b/source/extensions/quic_listeners/quiche/client_connection_factory_impl.h new file mode 100644 index 000000000000..14e83148e877 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/client_connection_factory_impl.h @@ -0,0 +1,61 @@ +#pragma once + +#include "common/http/http3/quic_client_connection_factory.h" +#include "common/http/http3/well_known_names.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" +#include "extensions/quic_listeners/quiche/envoy_quic_client_session.h" +#include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" +#include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h" +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/transport_sockets/tls/ssl_socket.h" + +#include "quiche/quic/core/http/quic_client_push_promise_index.h" +#include "quiche/quic/core/quic_utils.h" + +namespace Envoy { +namespace Quic { + +// TODO(#14829) we should avoid creating this per-connection. +struct QuicUpstreamData { + QuicUpstreamData(Event::Dispatcher& dispatcher, Envoy::Ssl::ClientContextConfig& config, + Network::Address::InstanceConstSharedPtr server_addr) + : conn_helper_(dispatcher), alarm_factory_(dispatcher, *conn_helper_.GetClock()), + config_(config), server_id_{config_.serverNameIndication(), + static_cast(server_addr->ip()->port()), false} {} + + EnvoyQuicConnectionHelper conn_helper_; + EnvoyQuicAlarmFactory alarm_factory_; + Envoy::Ssl::ClientContextConfig& config_; + quic::QuicServerId server_id_; + std::unique_ptr crypto_config_; + quic::ParsedQuicVersionVector supported_versions_{quic::CurrentSupportedVersions()}; +}; + +class EnvoyQuicClientSessionWithExtras : public EnvoyQuicClientSession { +public: + using EnvoyQuicClientSession::EnvoyQuicClientSession; + + std::unique_ptr quic_upstream_data_; +}; + +// A factory to create EnvoyQuicClientConnection instance for QUIC +class QuicClientConnectionFactoryImpl : public Http::QuicClientConnectionFactory { +public: + std::unique_ptr + createQuicNetworkConnection(Network::Address::InstanceConstSharedPtr server_addr, + Network::Address::InstanceConstSharedPtr local_addr, + Network::TransportSocketFactory& transport_socket_factory, + Stats::Scope& stats_scope, Event::Dispatcher& dispatcher, + TimeSource& time_source) override; + + std::string name() const override { return Http::QuicCodecNames::get().Quiche; } + + quic::QuicConfig quic_config_; + quic::QuicClientPushPromiseIndex push_promise_index_; +}; + +DECLARE_FACTORY(QuicClientConnectionFactoryImpl); + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/codec_impl.cc b/source/extensions/quic_listeners/quiche/codec_impl.cc index 730008129e59..51fad9e4bc20 100644 --- a/source/extensions/quic_listeners/quiche/codec_impl.cc +++ b/source/extensions/quic_listeners/quiche/codec_impl.cc @@ -113,7 +113,6 @@ QuicHttpServerConnectionFactoryImpl::createQuicServerConnection( REGISTER_FACTORY(QuicHttpClientConnectionFactoryImpl, Http::QuicHttpClientConnectionFactory); REGISTER_FACTORY(QuicHttpServerConnectionFactoryImpl, Http::QuicHttpServerConnectionFactory); -REGISTER_FACTORY(QuicClientConnectionFactoryImpl, Http::QuicClientConnectionFactory); } // namespace Quic } // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/codec_impl.h b/source/extensions/quic_listeners/quiche/codec_impl.h index 71909c7fc05c..8f655523c3cf 100644 --- a/source/extensions/quic_listeners/quiche/codec_impl.h +++ b/source/extensions/quic_listeners/quiche/codec_impl.h @@ -6,16 +6,8 @@ #include "common/http/http3/quic_codec_factory.h" #include "common/http/http3/well_known_names.h" -#include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" #include "extensions/quic_listeners/quiche/envoy_quic_client_session.h" -#include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" -#include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h" #include "extensions/quic_listeners/quiche/envoy_quic_server_session.h" -#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" -#include "extensions/transport_sockets/tls/ssl_socket.h" - -#include "quiche/quic/core/http/quic_client_push_promise_index.h" -#include "quiche/quic/core/quic_utils.h" namespace Envoy { namespace Quic { @@ -102,72 +94,8 @@ class QuicHttpServerConnectionFactoryImpl : public Http::QuicHttpServerConnectio std::string name() const override { return Http::QuicCodecNames::get().Quiche; } }; -// TODO(#14829) we should avoid creating this per-connection. -struct QuicUpstreamData { - QuicUpstreamData(Event::Dispatcher& dispatcher, Envoy::Ssl::ClientContextConfig& config, - Network::Address::InstanceConstSharedPtr server_addr) - : conn_helper_(dispatcher), alarm_factory_(dispatcher, *conn_helper_.GetClock()), - config_(config), server_id_{config_.serverNameIndication(), - static_cast(server_addr->ip()->port()), false} {} - - EnvoyQuicConnectionHelper conn_helper_; - EnvoyQuicAlarmFactory alarm_factory_; - Envoy::Ssl::ClientContextConfig& config_; - quic::QuicServerId server_id_; - std::unique_ptr crypto_config_; - quic::ParsedQuicVersionVector supported_versions_{quic::CurrentSupportedVersions()}; -}; - -class EnvoyQuicClientSessionWithExtras : public EnvoyQuicClientSession { -public: - using EnvoyQuicClientSession::EnvoyQuicClientSession; - - std::unique_ptr quic_upstream_data_; -}; - -// A factory to create EnvoyQuicClientConnection instance for QUIC -class QuicClientConnectionFactoryImpl : public Http::QuicClientConnectionFactory { -public: - std::unique_ptr - createQuicNetworkConnection(Network::Address::InstanceConstSharedPtr server_addr, - Network::Address::InstanceConstSharedPtr local_addr, - Network::TransportSocketFactory& transport_socket_factory, - Stats::Scope& stats_scope, Event::Dispatcher& dispatcher, - TimeSource& time_source) override { - // TODO(#14829): reject the config if a raw buffer socket is configured. - auto* ssl_socket_factory = - dynamic_cast( - &transport_socket_factory); - ASSERT(ssl_socket_factory != nullptr); - - std::unique_ptr upstream_data = - std::make_unique(dispatcher, *ssl_socket_factory->config(), server_addr); - upstream_data->crypto_config_ = std::make_unique( - std::make_unique(stats_scope, upstream_data->config_, time_source)); - - auto connection = std::make_unique( - quic::QuicUtils::CreateRandomConnectionId(), server_addr, upstream_data->conn_helper_, - upstream_data->alarm_factory_, - quic::ParsedQuicVersionVector{upstream_data->supported_versions_[0]}, local_addr, - dispatcher, nullptr); - auto ret = std::make_unique( - quic_config_, upstream_data->supported_versions_, std::move(connection), - upstream_data->server_id_, upstream_data->crypto_config_.get(), &push_promise_index_, - dispatcher, 0); - ret->Initialize(); - ret->quic_upstream_data_ = std::move(upstream_data); - return ret; - } - - quic::QuicConfig quic_config_; - std::string name() const override { return Http::QuicCodecNames::get().Quiche; } - - quic::QuicClientPushPromiseIndex push_promise_index_; -}; - DECLARE_FACTORY(QuicHttpClientConnectionFactoryImpl); DECLARE_FACTORY(QuicHttpServerConnectionFactoryImpl); -DECLARE_FACTORY(QuicClientConnectionFactoryImpl); } // namespace Quic } // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/integration/BUILD b/test/extensions/quic_listeners/quiche/integration/BUILD index dcd761caede9..5086389b21dc 100644 --- a/test/extensions/quic_listeners/quiche/integration/BUILD +++ b/test/extensions/quic_listeners/quiche/integration/BUILD @@ -23,6 +23,7 @@ envoy_cc_test( ], deps = [ "//source/extensions/quic_listeners/quiche:active_quic_listener_lib", + "//source/extensions/quic_listeners/quiche:client_connection_factory_lib", "//source/extensions/quic_listeners/quiche:quic_factory_lib", "//source/extensions/quic_listeners/quiche:quic_transport_socket_factory_lib", "//test/integration:protocol_integration_test_lib", diff --git a/test/integration/BUILD b/test/integration/BUILD index 292fa7a9b497..258bd28d24ca 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -695,6 +695,7 @@ envoy_cc_test_library( "//test/test_common:test_time_system_interface", "//test/test_common:utility_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", ], ) diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 4015fd135a7a..8aa9cc83113e 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -239,6 +239,17 @@ void BaseIntegrationTest::setUpstreamProtocol(FakeHttpConnection::Type protocol) config_helper_.configureUpstreamTls(false); config_helper_.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + // Docker doesn't allow writing to the v6 address returned by + // Network::Utility::getLocalAddress. + if (version_ == Network::Address::IpVersion::v6) { + auto* bind_config_address = bootstrap.mutable_static_resources() + ->mutable_clusters(0) + ->mutable_upstream_bind_config() + ->mutable_source_address(); + bind_config_address->set_address("::1"); + bind_config_address->set_port_value(0); + } + RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() >= 1, ""); ConfigHelper::HttpProtocolOptions protocol_options; protocol_options.mutable_explicit_http_config()->mutable_http3_protocol_options(); diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 989d16e9f6ac..411c1e6dd570 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -7,6 +7,7 @@ #include "envoy/api/api.h" #include "envoy/config/core/v3/base.pb.h" +#include "envoy/config/listener/v3/quic_config.pb.h" #include "envoy/grpc/status.h" #include "envoy/http/codec.h" #include "envoy/network/connection.h" From 37dc9455071eff12306c584834aaf2bbd1a56f19 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Mon, 22 Feb 2021 16:43:16 -0500 Subject: [PATCH 6/8] comments Signed-off-by: Alyssa Wilk --- source/common/http/http2/conn_pool.cc | 11 +++++++++++ source/common/http/http2/conn_pool.h | 3 +++ source/common/http/http3/conn_pool.cc | 12 ++++++------ source/common/http/http3/conn_pool.h | 7 ++++--- .../quiche/client_connection_factory_impl.cc | 17 ++++++++++------- .../quiche/client_connection_factory_impl.h | 8 ++++---- .../transport_sockets/tls/ssl_socket.h | 2 -- test/config/BUILD | 1 + test/config/utility.cc | 16 ++++++++++++---- test/config/utility.h | 2 +- test/integration/base_integration_test.cc | 2 +- 11 files changed, 53 insertions(+), 28 deletions(-) diff --git a/source/common/http/http2/conn_pool.cc b/source/common/http/http2/conn_pool.cc index 00d79eb3a762..b5f94c42d44c 100644 --- a/source/common/http/http2/conn_pool.cc +++ b/source/common/http/http2/conn_pool.cc @@ -64,6 +64,17 @@ MultiplexedActiveClientBase::MultiplexedActiveClientBase(HttpConnPoolImplBase& p cx_total.inc(); } +MultiplexedActiveClientBase::MultiplexedActiveClientBase(HttpConnPoolImplBase& parent, + Stats::Counter& cx_total, + Upstream::Host::CreateConnectionData& data) + : Envoy::Http::ActiveClient( + parent, maxStreamsPerConnection(parent.host()->cluster().maxRequestsPerConnection()), + parent.host()->cluster().http2Options().max_concurrent_streams().value(), data) { + codec_client_->setCodecClientCallbacks(*this); + codec_client_->setCodecConnectionCallbacks(*this); + cx_total.inc(); +} + MultiplexedActiveClientBase::MultiplexedActiveClientBase(Envoy::Http::HttpConnPoolImplBase& parent, Upstream::Host::CreateConnectionData& data, Stats::Counter& cx_total) diff --git a/source/common/http/http2/conn_pool.h b/source/common/http/http2/conn_pool.h index 51a25cbc051e..931e53798771 100644 --- a/source/common/http/http2/conn_pool.h +++ b/source/common/http/http2/conn_pool.h @@ -13,11 +13,14 @@ namespace Http { /** * Active client base for HTTP/2 and HTTP/3 */ +// TODO(#14829) move to source/common/http/conn_pool_base.h class MultiplexedActiveClientBase : public CodecClientCallbacks, public Http::ConnectionCallbacks, public Envoy::Http::ActiveClient { public: MultiplexedActiveClientBase(HttpConnPoolImplBase& parent, Stats::Counter& cx_total); + MultiplexedActiveClientBase(HttpConnPoolImplBase& parent, Stats::Counter& cx_total, + Upstream::Host::CreateConnectionData& data); ~MultiplexedActiveClientBase() override = default; // ConnPoolImpl::ActiveClient diff --git a/source/common/http/http3/conn_pool.cc b/source/common/http/http3/conn_pool.cc index 9165d971cb8c..f314725ef71a 100644 --- a/source/common/http/http3/conn_pool.cc +++ b/source/common/http/http3/conn_pool.cc @@ -25,11 +25,9 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ Upstream::ClusterConnectivityState& state, TimeSource& time_source) { return std::make_unique( host, priority, dispatcher, options, transport_socket_options, random_generator, state, - [](HttpConnPoolImplBase* pool) { return std::make_unique(*pool); }, - [&dispatcher, &time_source](Upstream::Host::CreateConnectionData& data, - HttpConnPoolImplBase* pool) { - // TODO(#14829) this is creating CreateConnectionData and overwriting. Refactor the base - // pool class to inject the correct CreateConnectionData and avoid the wasted allocation. + [&dispatcher, &time_source](HttpConnPoolImplBase* pool) { + Upstream::Host::CreateConnectionData data{}; + data.host_description_ = pool->host(); auto host_address = data.host_description_->address(); auto source_address = data.host_description_->cluster().sourceAddress(); if (!source_address.get()) { @@ -37,13 +35,15 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ } Network::TransportSocketFactory& transport_socket_factory = data.host_description_->transportSocketFactory(); - data.connection_->close(Network::ConnectionCloseType::NoFlush); data.connection_ = Config::Utility::getAndCheckFactoryByName( Http::QuicCodecNames::get().Quiche) .createQuicNetworkConnection(host_address, source_address, transport_socket_factory, data.host_description_->cluster().statsScope(), dispatcher, time_source); + return std::make_unique(*pool, data); + }, + [](Upstream::Host::CreateConnectionData& data, HttpConnPoolImplBase* pool) { CodecClientPtr codec{new CodecClientProd( CodecClient::Type::HTTP3, std::move(data.connection_), data.host_description_, pool->dispatcher(), pool->randomGenerator())}; diff --git a/source/common/http/http3/conn_pool.h b/source/common/http/http3/conn_pool.h index 988cc176a969..df9a6832c5a6 100644 --- a/source/common/http/http3/conn_pool.h +++ b/source/common/http/http3/conn_pool.h @@ -15,9 +15,10 @@ namespace Http3 { // connection based on HTTP/2 config. Sort out the HTTP/3 config story. class ActiveClient : public MultiplexedActiveClientBase { public: - ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent) - : MultiplexedActiveClientBase(parent, - parent.host()->cluster().stats().upstream_cx_http3_total_) {} + ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent, + Upstream::Host::CreateConnectionData& data) + : MultiplexedActiveClientBase( + parent, parent.host()->cluster().stats().upstream_cx_http3_total_, data) {} }; ConnectionPool::InstancePtr diff --git a/source/extensions/quic_listeners/quiche/client_connection_factory_impl.cc b/source/extensions/quic_listeners/quiche/client_connection_factory_impl.cc index 75d0fa6e0059..9b8f40e34c73 100644 --- a/source/extensions/quic_listeners/quiche/client_connection_factory_impl.cc +++ b/source/extensions/quic_listeners/quiche/client_connection_factory_impl.cc @@ -1,5 +1,7 @@ #include "extensions/quic_listeners/quiche/client_connection_factory_impl.h" +#include "extensions/quic_listeners/quiche/quic_transport_socket_factory.h" + namespace Envoy { namespace Quic { @@ -9,16 +11,17 @@ QuicClientConnectionFactoryImpl::createQuicNetworkConnection( Network::Address::InstanceConstSharedPtr local_addr, Network::TransportSocketFactory& transport_socket_factory, Stats::Scope& stats_scope, Event::Dispatcher& dispatcher, TimeSource& time_source) { - // TODO(#14829): reject the config if a raw buffer socket is configured. - auto* ssl_socket_factory = - dynamic_cast( - &transport_socket_factory); - ASSERT(ssl_socket_factory != nullptr); + // TODO(#14829): reject config if anything but QuicClientTransportSocketConfigFactory configured. + // raw buffer socket is configured. + auto* quic_socket_factory = + dynamic_cast(&transport_socket_factory); + ASSERT(quic_socket_factory != nullptr); + const auto& client_context_config = quic_socket_factory->clientContextConfig(); std::unique_ptr upstream_data = - std::make_unique(dispatcher, *ssl_socket_factory->config(), server_addr); + std::make_unique(dispatcher, client_context_config, server_addr); upstream_data->crypto_config_ = std::make_unique( - std::make_unique(stats_scope, upstream_data->config_, time_source)); + std::make_unique(stats_scope, client_context_config, time_source)); auto connection = std::make_unique( quic::QuicUtils::CreateRandomConnectionId(), server_addr, upstream_data->conn_helper_, diff --git a/source/extensions/quic_listeners/quiche/client_connection_factory_impl.h b/source/extensions/quic_listeners/quiche/client_connection_factory_impl.h index 14e83148e877..8e9b4fe2511d 100644 --- a/source/extensions/quic_listeners/quiche/client_connection_factory_impl.h +++ b/source/extensions/quic_listeners/quiche/client_connection_factory_impl.h @@ -17,16 +17,16 @@ namespace Envoy { namespace Quic { // TODO(#14829) we should avoid creating this per-connection. +// Figure out what goes in per-cluster data, and what is per-connection and clean up. struct QuicUpstreamData { - QuicUpstreamData(Event::Dispatcher& dispatcher, Envoy::Ssl::ClientContextConfig& config, + QuicUpstreamData(Event::Dispatcher& dispatcher, const Envoy::Ssl::ClientContextConfig& config, Network::Address::InstanceConstSharedPtr server_addr) : conn_helper_(dispatcher), alarm_factory_(dispatcher, *conn_helper_.GetClock()), - config_(config), server_id_{config_.serverNameIndication(), - static_cast(server_addr->ip()->port()), false} {} + server_id_{config.serverNameIndication(), static_cast(server_addr->ip()->port()), + false} {} EnvoyQuicConnectionHelper conn_helper_; EnvoyQuicAlarmFactory alarm_factory_; - Envoy::Ssl::ClientContextConfig& config_; quic::QuicServerId server_id_; std::unique_ptr crypto_config_; quic::ParsedQuicVersionVector supported_versions_{quic::CurrentSupportedVersions()}; diff --git a/source/extensions/transport_sockets/tls/ssl_socket.h b/source/extensions/transport_sockets/tls/ssl_socket.h index 0a1292ad557b..cb15d82fa935 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.h +++ b/source/extensions/transport_sockets/tls/ssl_socket.h @@ -114,8 +114,6 @@ class ClientSslSocketFactory : public Network::TransportSocketFactory, // Secret::SecretCallbacks void onAddOrUpdateSecret() override; - Envoy::Ssl::ClientContextConfigPtr& config() { return config_; } - private: Envoy::Ssl::ContextManager& manager_; Stats::Scope& stats_scope_; diff --git a/test/config/BUILD b/test/config/BUILD index 1f9d463b5b03..fe8cf3472c9a 100644 --- a/test/config/BUILD +++ b/test/config/BUILD @@ -37,6 +37,7 @@ envoy_cc_test_library( "@envoy_api//envoy/config/tap/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/access_loggers/file/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/transport_sockets/quic/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/transport_sockets/tap/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/upstreams/http/v3:pkg_cc_proto", diff --git a/test/config/utility.cc b/test/config/utility.cc index bba573e5cdf2..8aac6793c11f 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -9,6 +9,7 @@ #include "envoy/config/tap/v3/common.pb.h" #include "envoy/extensions/access_loggers/file/v3/file.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" +#include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.h" #include "envoy/extensions/transport_sockets/tap/v3/tap.pb.h" #include "envoy/extensions/transport_sockets/tls/v3/cert.pb.h" #include "envoy/http/codec.h" @@ -666,8 +667,8 @@ void ConfigHelper::applyConfigModifiers() { config_modifiers_.clear(); } -void ConfigHelper::configureUpstreamTls(bool use_alpn) { - addConfigModifier([use_alpn](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { +void ConfigHelper::configureUpstreamTls(bool use_alpn, bool http3) { + addConfigModifier([use_alpn, http3](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { auto* cluster = bootstrap.mutable_static_resources()->mutable_clusters(0); ConfigHelper::HttpProtocolOptions protocol_options; @@ -704,8 +705,15 @@ void ConfigHelper::configureUpstreamTls(bool use_alpn) { TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); // The test certs are for *.lyft.com, so make sure SNI matches. tls_context.set_sni("foo.lyft.com"); - cluster->mutable_transport_socket()->set_name("envoy.transport_sockets.tls"); - cluster->mutable_transport_socket()->mutable_typed_config()->PackFrom(tls_context); + if (http3) { + cluster->mutable_transport_socket()->set_name("envoy.transport_sockets.quic"); + envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport quic_context; + quic_context.mutable_upstream_tls_context()->CopyFrom(tls_context); + cluster->mutable_transport_socket()->mutable_typed_config()->PackFrom(quic_context); + } else { + cluster->mutable_transport_socket()->set_name("envoy.transport_sockets.tls"); + cluster->mutable_transport_socket()->mutable_typed_config()->PackFrom(tls_context); + } }); } diff --git a/test/config/utility.h b/test/config/utility.h index 7bf60d63b1c1..6d6777831112 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -259,7 +259,7 @@ class ConfigHelper { void applyConfigModifiers(); // Configure Envoy to do TLS to upstream. - void configureUpstreamTls(bool use_alpn = false); + void configureUpstreamTls(bool use_alpn = false, bool http3 = false); // Skip validation that ensures that all upstream ports are referenced by the // configuration generated in ConfigHelper::finalize. diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 8aa9cc83113e..3b8f00aa2892 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -236,7 +236,7 @@ void BaseIntegrationTest::setUpstreamProtocol(FakeHttpConnection::Type protocol) RELEASE_ASSERT(protocol == FakeHttpConnection::Type::HTTP3, ""); setUdpFakeUpstream(true); upstream_tls_ = true; - config_helper_.configureUpstreamTls(false); + config_helper_.configureUpstreamTls(false, true); config_helper_.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { // Docker doesn't allow writing to the v6 address returned by From d95ca6f64f7f625275bf795a77e4f0201b98dd0d Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Wed, 24 Feb 2021 13:12:56 -0500 Subject: [PATCH 7/8] comments Signed-off-by: Alyssa Wilk --- source/common/http/http3/quic_client_connection_factory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/http/http3/quic_client_connection_factory.h b/source/common/http/http3/quic_client_connection_factory.h index 1e839b9fb020..7c38ca513946 100644 --- a/source/common/http/http3/quic_client_connection_factory.h +++ b/source/common/http/http3/quic_client_connection_factory.h @@ -10,7 +10,7 @@ namespace Envoy { namespace Http { -// A factory to create EnvoyQuicClientConnection instance for QUIC +// A factory to create EnvoyQuicClientSession and EnvoyQuicClientConnection for QUIC class QuicClientConnectionFactory : public Config::UntypedFactory { public: ~QuicClientConnectionFactory() override = default; From 9811ae24e33293da85f22e6d48fca5f43b52fc7a Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Tue, 2 Mar 2021 15:52:43 -0500 Subject: [PATCH 8/8] oops Signed-off-by: Alyssa Wilk --- test/integration/fake_upstream.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 7a5023a4c0a6..e79d5d1cd298 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -721,7 +721,6 @@ testing::AssertionResult FakeUpstream::waitForUdpDatagram(Network::UdpRecvData& } void FakeUpstream::onRecvDatagram(Network::UdpRecvData& data) { - std::cerr << "Got datagrams. Need to dispatch\n"; absl::MutexLock lock(&lock_); received_datagrams_.emplace_back(std::move(data)); }