quic: HTTP/3 upstream initial checkin

include/envoy/upstream/cluster_manager.h

@@ -356,7 +356,7 @@ class ClusterManagerFactory {
                    ResourcePriority priority, std::vector<Http::Protocol>& protocol,
                    const Network::ConnectionSocket::OptionsSharedPtr& options,
                    const Network::TransportSocketOptionsSharedPtr& transport_socket_options,
-                   ClusterConnectivityState& state) PURE;
+                   TimeSource& time_source, ClusterConnectivityState& state) PURE;
 
   /**
    * Allocate a TCP connection pool for the host. Pools are separated by 'priority' and

include/envoy/upstream/upstream.h

@@ -539,6 +539,7 @@ class PrioritySet {
   COUNTER(upstream_cx_destroy_with_active_rq)                                                      \
   COUNTER(upstream_cx_http1_total)                                                                 \
   COUNTER(upstream_cx_http2_total)                                                                 \
+  COUNTER(upstream_cx_http3_total)                                                                 \
   COUNTER(upstream_cx_idle_timeout)                                                                \
   COUNTER(upstream_cx_max_requests)                                                                \
   COUNTER(upstream_cx_none_healthy)                                                                \

source/common/http/conn_pool_base.cc

@@ -31,8 +31,7 @@ wrapTransportSocketOptions(Network::TransportSocketOptionsSharedPtr transport_so
       fallbacks.push_back(Http::Utility::AlpnNames::get().Http2);
       break;
     case Http::Protocol::Http3:
-      // TODO(snowp): Add once HTTP/3 upstream support is added.
-      NOT_IMPLEMENTED_GCOVR_EXCL_LINE;
+      // TODO(#14829) hard-code H3 ALPN, consider failing if other things are negotiated.
       break;
     }
   }

source/common/http/http2/conn_pool.cc

@@ -10,13 +10,12 @@
 
 namespace Envoy {
 namespace Http {
-namespace Http2 {
 
 // All streams are 2^31. Client streams are half that, minus stream 0. Just to be on the safe
 // side we do 2^29.
 static const uint64_t DEFAULT_MAX_STREAMS = (1 << 29);
 
-void ActiveClient::onGoAway(Http::GoAwayErrorCode) {
+void MultiplexedActiveClientBase::onGoAway(Http::GoAwayErrorCode) {
   ENVOY_CONN_LOG(debug, "remote goaway", *codec_client_);
   parent_.host()->cluster().stats().upstream_cx_close_notify_.inc();
   if (state_ != ActiveClient::State::DRAINING) {
@@ -28,7 +27,7 @@ void ActiveClient::onGoAway(Http::GoAwayErrorCode) {
   }
 }
 
-void ActiveClient::onStreamDestroy() {
+void MultiplexedActiveClientBase::onStreamDestroy() {
   parent().onStreamClosed(*this, false);
 
   // If we are destroying this stream because of a disconnect, do not check for drain here. We will
@@ -39,7 +38,7 @@ void ActiveClient::onStreamDestroy() {
   }
 }
 
-void ActiveClient::onStreamReset(Http::StreamResetReason reason) {
+void MultiplexedActiveClientBase::onStreamReset(Http::StreamResetReason reason) {
   if (reason == StreamResetReason::ConnectionTermination ||
       reason == StreamResetReason::ConnectionFailure) {
     parent_.host()->cluster().stats().upstream_rq_pending_failure_eject_.inc();
@@ -55,31 +54,56 @@ uint64_t maxStreamsPerConnection(uint64_t max_streams_config) {
   return (max_streams_config != 0) ? max_streams_config : DEFAULT_MAX_STREAMS;
 }
 
-ActiveClient::ActiveClient(HttpConnPoolImplBase& parent)
+MultiplexedActiveClientBase::MultiplexedActiveClientBase(HttpConnPoolImplBase& parent,
+                                                         Stats::Counter& cx_total)
     : Envoy::Http::ActiveClient(
           parent, maxStreamsPerConnection(parent.host()->cluster().maxRequestsPerConnection()),
           parent.host()->cluster().http2Options().max_concurrent_streams().value()) {
   codec_client_->setCodecClientCallbacks(*this);
   codec_client_->setCodecConnectionCallbacks(*this);
-  parent.host()->cluster().stats().upstream_cx_http2_total_.inc();
+  cx_total.inc();
 }
 
-ActiveClient::ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
-                           Upstream::Host::CreateConnectionData& data)
+MultiplexedActiveClientBase::MultiplexedActiveClientBase(HttpConnPoolImplBase& parent,
+                                                         Stats::Counter& cx_total,
+                                                         Upstream::Host::CreateConnectionData& data)
+    : Envoy::Http::ActiveClient(
+          parent, maxStreamsPerConnection(parent.host()->cluster().maxRequestsPerConnection()),
+          parent.host()->cluster().http2Options().max_concurrent_streams().value(), data) {
+  codec_client_->setCodecClientCallbacks(*this);
+  codec_client_->setCodecConnectionCallbacks(*this);
+  cx_total.inc();
+}
+
+MultiplexedActiveClientBase::MultiplexedActiveClientBase(Envoy::Http::HttpConnPoolImplBase& parent,
+                                                         Upstream::Host::CreateConnectionData& data,
+                                                         Stats::Counter& cx_total)
     : Envoy::Http::ActiveClient(
           parent, maxStreamsPerConnection(parent.host()->cluster().maxRequestsPerConnection()),
           parent.host()->cluster().http2Options().max_concurrent_streams().value(), data) {
   codec_client_->setCodecClientCallbacks(*this);
   codec_client_->setCodecConnectionCallbacks(*this);
-  parent.host()->cluster().stats().upstream_cx_http2_total_.inc();
+  cx_total.inc();
 }
 
-bool ActiveClient::closingWithIncompleteStream() const { return closed_with_active_rq_; }
+bool MultiplexedActiveClientBase::closingWithIncompleteStream() const {
+  return closed_with_active_rq_;
+}
 
-RequestEncoder& ActiveClient::newStreamEncoder(ResponseDecoder& response_decoder) {
+RequestEncoder& MultiplexedActiveClientBase::newStreamEncoder(ResponseDecoder& response_decoder) {
   return codec_client_->newStream(response_decoder);
 }
 
+namespace Http2 {
+ActiveClient::ActiveClient(HttpConnPoolImplBase& parent)
+    : MultiplexedActiveClientBase(parent,
+                                  parent.host()->cluster().stats().upstream_cx_http2_total_) {}
+
+ActiveClient::ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
+                           Upstream::Host::CreateConnectionData& data)
+    : MultiplexedActiveClientBase(parent, data,
+                                  parent.host()->cluster().stats().upstream_cx_http2_total_) {}
+
 ConnectionPool::InstancePtr
 allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator,
                  Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,

source/common/http/http2/conn_pool.h

@@ -9,19 +9,19 @@
 
 namespace Envoy {
 namespace Http {
-namespace Http2 {
 
 /**
- * Implementation of an active client for HTTP/2
+ * Active client base for HTTP/2 and HTTP/3
  */
-class ActiveClient : public CodecClientCallbacks,
-                     public Http::ConnectionCallbacks,
-                     public Envoy::Http::ActiveClient {
+// TODO(#14829) move to source/common/http/conn_pool_base.h
+class MultiplexedActiveClientBase : public CodecClientCallbacks,
+                                    public Http::ConnectionCallbacks,
+                                    public Envoy::Http::ActiveClient {
 public:
-  ActiveClient(HttpConnPoolImplBase& parent);
-  ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
-               Upstream::Host::CreateConnectionData& data);
-  ~ActiveClient() override = default;
+  MultiplexedActiveClientBase(HttpConnPoolImplBase& parent, Stats::Counter& cx_total);
+  MultiplexedActiveClientBase(HttpConnPoolImplBase& parent, Stats::Counter& cx_total,
+                              Upstream::Host::CreateConnectionData& data);
+  ~MultiplexedActiveClientBase() override = default;
 
   // ConnPoolImpl::ActiveClient
   bool closingWithIncompleteStream() const override;
@@ -34,9 +34,26 @@ class ActiveClient : public CodecClientCallbacks,
   // Http::ConnectionCallbacks
   void onGoAway(Http::GoAwayErrorCode error_code) override;
 
+protected:
+  MultiplexedActiveClientBase(Envoy::Http::HttpConnPoolImplBase& parent,
+                              Upstream::Host::CreateConnectionData& data, Stats::Counter& cx_total);
+
+private:
   bool closed_with_active_rq_{};
 };
 
+namespace Http2 {
+
+/**
+ * Implementation of an active client for HTTP/2
+ */
+class ActiveClient : public MultiplexedActiveClientBase {
+public:
+  ActiveClient(HttpConnPoolImplBase& parent);
+  ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
+               Upstream::Host::CreateConnectionData& data);
+};
+
 ConnectionPool::InstancePtr
 allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator,
                  Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,

source/common/http/http3/BUILD

@@ -8,6 +8,20 @@ licenses(["notice"])  # Apache 2
 
 envoy_package()
 
+envoy_cc_library(
+    name = "conn_pool_lib",
+    srcs = ["conn_pool.cc"],
+    hdrs = ["conn_pool.h"],
+    deps = [
+        ":quic_client_connection_factory_lib",
+        "//include/envoy/event:dispatcher_interface",
+        "//include/envoy/upstream:upstream_interface",
+        "//source/common/http:codec_client_lib",
+        "//source/common/http:conn_pool_base_lib",
+        "//source/common/http/http2:conn_pool_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "quic_codec_factory_lib",
     hdrs = ["quic_codec_factory.h"],
@@ -18,6 +32,17 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "quic_client_connection_factory_lib",
+    hdrs = ["quic_client_connection_factory.h"],
+    deps = [
+        "//include/envoy/config:typed_config_interface",
+        "//include/envoy/network:connection_interface",
+        "//include/envoy/ssl:context_config_interface",
+        "@envoy_api//envoy/config/listener/v3:pkg_cc_proto",
+    ],
+)
+
 envoy_cc_library(
     name = "well_known_names",
     hdrs = ["well_known_names.h"],

source/common/http/http3/conn_pool.cc

@@ -0,0 +1,57 @@
+#include "common/http/http3/conn_pool.h"
+
+#include <cstdint>
+
+#include "envoy/event/dispatcher.h"
+#include "envoy/upstream/upstream.h"
+
+#include "common/config/utility.h"
+#include "common/http/http3/quic_client_connection_factory.h"
+#include "common/http/http3/well_known_names.h"
+#include "common/http/utility.h"
+#include "common/network/address_impl.h"
+#include "common/network/utility.h"
+#include "common/runtime/runtime_features.h"
+
+namespace Envoy {
+namespace Http {
+namespace Http3 {
+
+ConnectionPool::InstancePtr
+allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator,
+                 Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,
+                 const Network::ConnectionSocket::OptionsSharedPtr& options,
+                 const Network::TransportSocketOptionsSharedPtr& transport_socket_options,
+                 Upstream::ClusterConnectivityState& state, TimeSource& time_source) {
+  return std::make_unique<FixedHttpConnPoolImpl>(
+      host, priority, dispatcher, options, transport_socket_options, random_generator, state,
+      [&dispatcher, &time_source](HttpConnPoolImplBase* pool) {
+        Upstream::Host::CreateConnectionData data{};
+        data.host_description_ = pool->host();
+        auto host_address = data.host_description_->address();
+        auto source_address = data.host_description_->cluster().sourceAddress();
+        if (!source_address.get()) {
+          source_address = Network::Utility::getLocalAddress(host_address->ip()->version());
+        }
+        Network::TransportSocketFactory& transport_socket_factory =
+            data.host_description_->transportSocketFactory();
+        data.connection_ =
+            Config::Utility::getAndCheckFactoryByName<Http::QuicClientConnectionFactory>(
+                Http::QuicCodecNames::get().Quiche)
+                .createQuicNetworkConnection(host_address, source_address, transport_socket_factory,
+                                             data.host_description_->cluster().statsScope(),
+                                             dispatcher, time_source);
+        return std::make_unique<ActiveClient>(*pool, data);
+      },
+      [](Upstream::Host::CreateConnectionData& data, HttpConnPoolImplBase* pool) {
+        CodecClientPtr codec{new CodecClientProd(
+            CodecClient::Type::HTTP3, std::move(data.connection_), data.host_description_,
+            pool->dispatcher(), pool->randomGenerator())};
+        return codec;
+      },
+      std::vector<Protocol>{Protocol::Http3});
+}
+
+} // namespace Http3
+} // namespace Http
+} // namespace Envoy

source/common/http/http3/conn_pool.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <cstdint>
+
+#include "envoy/upstream/upstream.h"
+
+#include "common/http/codec_client.h"
+#include "common/http/http2/conn_pool.h"
+
+namespace Envoy {
+namespace Http {
+namespace Http3 {
+
+// TODO(#14829) the constructor of Http2::ActiveClient sets max requests per
+// connection based on HTTP/2 config. Sort out the HTTP/3 config story.
+class ActiveClient : public MultiplexedActiveClientBase {
+public:
+  ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
+               Upstream::Host::CreateConnectionData& data)
+      : MultiplexedActiveClientBase(
+            parent, parent.host()->cluster().stats().upstream_cx_http3_total_, data) {}
+};
+
+ConnectionPool::InstancePtr
+allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator,
+                 Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,
+                 const Network::ConnectionSocket::OptionsSharedPtr& options,
+                 const Network::TransportSocketOptionsSharedPtr& transport_socket_options,
+                 Upstream::ClusterConnectivityState& state, TimeSource& time_source);
+
+} // namespace Http3
+} // namespace Http
+} // namespace Envoy

source/common/http/http3/quic_client_connection_factory.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <string>
+
+#include "envoy/config/listener/v3/quic_config.pb.h"
+#include "envoy/config/typed_config.h"
+#include "envoy/network/connection.h"
+#include "envoy/ssl/context_config.h"
+
+namespace Envoy {
+namespace Http {
+
+// A factory to create EnvoyQuicClientConnection instance for QUIC
+class QuicClientConnectionFactory : public Config::UntypedFactory {
+public:
+  ~QuicClientConnectionFactory() override = default;
+
+  virtual std::unique_ptr<Network::ClientConnection>
+  createQuicNetworkConnection(Network::Address::InstanceConstSharedPtr server_addr,
+                              Network::Address::InstanceConstSharedPtr local_addr,
+                              Network::TransportSocketFactory& transport_socket_factory,
+                              Stats::Scope& stats_scope, Event::Dispatcher& dispatcher,
+                              TimeSource& time_source) PURE;
+
+  std::string category() const override { return "envoy.quic_connection"; }
+};
+
+} // namespace Http
+} // namespace Envoy

source/common/upstream/BUILD

@@ -61,6 +61,7 @@ envoy_cc_library(
         "//source/common/http:mixed_conn_pool",
         "//source/common/http/http1:conn_pool_lib",
         "//source/common/http/http2:conn_pool_lib",
+        "//source/common/http/http3:conn_pool_lib",
         "//source/common/network:resolver_lib",
         "//source/common/network:utility_lib",
         "//source/common/protobuf:utility_lib",

source/common/upstream/cluster_manager_impl.cc

@@ -28,6 +28,7 @@
 #include "common/http/async_client_impl.h"
 #include "common/http/http1/conn_pool.h"
 #include "common/http/http2/conn_pool.h"
+#include "common/http/http3/conn_pool.h"
 #include "common/http/mixed_conn_pool.h"
 #include "common/network/resolver_impl.h"
 #include "common/network/utility.h"
@@ -1419,7 +1420,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::connPool(
             parent_.thread_local_dispatcher_, host, priority, upstream_protocols,
             !upstream_options->empty() ? upstream_options : nullptr,
             have_transport_socket_options ? context->upstreamTransportSocketOptions() : nullptr,
-            parent_.cluster_manager_state_);
+            parent_.parent_.time_source_, parent_.cluster_manager_state_);
       });
 
   if (pool.has_value()) {
@@ -1488,7 +1489,7 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool(
     Event::Dispatcher& dispatcher, HostConstSharedPtr host, ResourcePriority priority,
     std::vector<Http::Protocol>& protocols,
     const Network::ConnectionSocket::OptionsSharedPtr& options,
-    const Network::TransportSocketOptionsSharedPtr& transport_socket_options,
+    const Network::TransportSocketOptionsSharedPtr& transport_socket_options, TimeSource& source,
     ClusterConnectivityState& state) {
   if (protocols.size() == 2) {
     ASSERT((protocols[0] == Http::Protocol::Http2 && protocols[1] == Http::Protocol::Http11) ||
@@ -1503,6 +1504,11 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool(
     return Http::Http2::allocateConnPool(dispatcher, api_.randomGenerator(), host, priority,
                                          options, transport_socket_options, state);
   }
+  if (protocols.size() == 1 && protocols[0] == Http::Protocol::Http3 &&
+      runtime_.snapshot().featureEnabled("upstream.use_http3", 100)) {
+    return Http::Http3::allocateConnPool(dispatcher, api_.randomGenerator(), host, priority,
+                                         options, transport_socket_options, state, source);
+  }
   ASSERT(protocols.size() == 1 && protocols[0] == Http::Protocol::Http11);
   return Http::Http1::allocateConnPool(dispatcher, api_.randomGenerator(), host, priority, options,
                                        transport_socket_options, state);

source/common/upstream/cluster_manager_impl.h

@@ -64,7 +64,7 @@ class ProdClusterManagerFactory : public ClusterManagerFactory {
                    ResourcePriority priority, std::vector<Http::Protocol>& protocol,
                    const Network::ConnectionSocket::OptionsSharedPtr& options,
                    const Network::TransportSocketOptionsSharedPtr& transport_socket_options,
-                   ClusterConnectivityState& state) override;
+                   TimeSource& time_source, ClusterConnectivityState& state) override;
   Tcp::ConnectionPool::InstancePtr
   allocateTcpConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host,
                       ResourcePriority priority,