http: alpn upstream

api/envoy/config/cluster/v3/cluster.proto

@@ -135,9 +135,12 @@ message Cluster {
   }
 
   enum ClusterProtocolSelection {
-    // Cluster can only operate on one of the possible upstream protocols (HTTP1.1, HTTP2).
-    // If :ref:`http2_protocol_options <envoy_api_field_config.cluster.v3.Cluster.http2_protocol_options>` are
-    // present, HTTP2 will be used, otherwise HTTP1.1 will be used.
+    // If both :ref:`http2_protocol_options <envoy_api_field_config.cluster.v3.Cluster.http2_protocol_options>`
+    // and :ref:`http_protocol_options <envoy_api_field_config.cluster.v3.Cluster.http_protocol_options>` are
+    // configured, Envoy will attempt to do ALPN negotiation for TLS connections, failing
+    // over to HTTP/1.1 if ALPN negotiation fails.
+    // If only one protocol option is present it will be used as the hard-coded
+    // protocol. If neither is present, HTTP/1.1 will be used.
     USE_CONFIGURED_PROTOCOL = 0;
 
     // Use HTTP1.1 or HTTP2, depending on which one is used on the downstream connection.

docs/root/version_history/current.rst

@@ -49,6 +49,7 @@ New Features
 * hds: added support for delta updates in the :ref:`HealthCheckSpecifier <envoy_v3_api_msg_service.health.v3.HealthCheckSpecifier>`, making only the Endpoints and Health Checkers that changed be reconstructed on receiving a new message, rather than the entire HDS.
 * health_check: added option to use :ref:`no_traffic_healthy_interval <envoy_v3_api_field_config.core.v3.HealthCheck.no_traffic_healthy_interval>` which allows a different no traffic interval when the host is healthy.
 * http: added frame flood and abuse checks to the upstream HTTP/2 codec. This check is off by default and can be enabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to true.
+* http: alpn is now supported upstream, configurable by setting both :ref:`HTTP/1 options <envoy_v3_api_msg_config.core.v3.HttpProtocolOptions>` and :ref:`HTTP/2 options <envoy_v3_api_msg_config.core.v3.Http2ProtocolOptions>` for a given cluster.
 * jwt_authn: added support for :ref:`per-route config <envoy_v3_api_msg_extensions.filters.http.jwt_authn.v3.PerRouteConfig>`.
 * listener: added an optional :ref:`default filter chain <envoy_v3_api_field_config.listener.v3.Listener.default_filter_chain>`. If this field is supplied, and none of the :ref:`filter_chains <envoy_v3_api_field_config.listener.v3.Listener.filter_chains>` matches, this default filter chain is used to serve the connection.
 * lua: added `downstreamDirectRemoteAddress()` and `downstreamLocalAddress()` APIs to :ref:`streamInfo() <config_http_filters_lua_stream_info_wrapper>`.

include/envoy/upstream/cluster_manager.h

@@ -316,7 +316,7 @@ class ClusterManagerFactory {
    */
   virtual Http::ConnectionPool::InstancePtr
   allocateConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host,
-                   ResourcePriority priority, Http::Protocol protocol,
+                   ResourcePriority priority, std::vector<Http::Protocol>& protocol,
                    const Network::ConnectionSocket::OptionsSharedPtr& options,
                    const Network::TransportSocketOptionsSharedPtr& transport_socket_options) PURE;
 

include/envoy/upstream/upstream.h

@@ -712,6 +712,10 @@ class ClusterInfo {
     static const uint64_t USE_DOWNSTREAM_PROTOCOL = 0x2;
     // Whether connections should be immediately closed upon health failure.
     static const uint64_t CLOSE_CONNECTIONS_ON_HOST_HEALTH_FAILURE = 0x4;
+    // If HTTP2 is true, the upstream protocol will be negotiated using ALPN.
+    // If ALPN is attempted but not supported by the upstream (non-TLS or simply not
+    // negotiated) HTTP/1.1 is used.
+    static const uint64_t USE_ALPN = 0x8;
   };
 
   virtual ~ClusterInfo() = default;
@@ -962,9 +966,9 @@ class ClusterInfo {
   virtual void createNetworkFilterChain(Network::Connection& connection) const PURE;
 
   /**
-   * Calculate upstream protocol based on features.
+   * Calculate upstream protocol(s) based on features.
    */
-  virtual Http::Protocol
+  virtual std::vector<Http::Protocol>
   upstreamHttpProtocol(absl::optional<Http::Protocol> downstream_protocol) const PURE;
 
   /**

source/common/conn_pool/conn_pool_base.cc

@@ -308,6 +308,11 @@ void ConnPoolImplBase::onConnectionEvent(ActiveClient& client, absl::string_view
     connecting_stream_capacity_ -= client.effectiveConcurrentStreamLimit();
   }
 
+  if (client.connect_timer_) {
+    client.connect_timer_->disableTimer();
+    client.connect_timer_.reset();
+  }
+
   if (event == Network::ConnectionEvent::RemoteClose ||
       event == Network::ConnectionEvent::LocalClose) {
     // The client died.
@@ -363,18 +368,15 @@ void ConnPoolImplBase::onConnectionEvent(ActiveClient& client, absl::string_view
   } else if (event == Network::ConnectionEvent::Connected) {
     client.conn_connect_ms_->complete();
     client.conn_connect_ms_.reset();
-
     ASSERT(client.state_ == ActiveClient::State::CONNECTING);
     transitionActiveClientState(client, ActiveClient::State::READY);
 
+    // At this point for the mixed ALPN pool client may be deleted. Do not
+    // refer to client after this point.
+    onConnected(client);
     onUpstreamReady();
     checkForDrained();
   }
-
-  if (client.connect_timer_) {
-    client.connect_timer_->disableTimer();
-    client.connect_timer_.reset();
-  }
 }
 
 PendingStream::PendingStream(ConnPoolImplBase& parent) : parent_(parent) {

source/common/conn_pool/conn_pool_base.h

@@ -48,6 +48,8 @@ class ActiveClient : public LinkedObject<ActiveClient>,
     return std::min(remaining_streams_, concurrent_stream_limit_);
   }
 
+  // Returns the application protocol, or absl::nullopt for TCP.
+  virtual absl::optional<Http::Protocol> protocol() const PURE;
   // Closes the underlying connection.
   virtual void close() PURE;
   // Returns the ID of the underlying connection.
@@ -177,6 +179,8 @@ class ConnPoolImplBase : protected Logger::Loggable<Logger::Id::pool> {
   bool hasPendingStreams() const { return !pending_streams_.empty(); }
 
 protected:
+  virtual void onConnected(Envoy::ConnectionPool::ActiveClient&) {}
+
   // Creates up to 3 connections, based on the prefetch ratio.
   void tryCreateNewConnections();
 

source/common/http/BUILD

@@ -140,6 +140,18 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "mixed_conn_pool",
+    srcs = ["mixed_conn_pool.cc"],
+    hdrs = ["mixed_conn_pool.h"],
+    deps = [
+        ":conn_pool_base_lib",
+        "//source/common/http/http1:conn_pool_lib",
+        "//source/common/http/http2:conn_pool_lib",
+        "//source/common/tcp:conn_pool_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "conn_manager_config_interface",
     hdrs = ["conn_manager_config.h"],

source/common/http/codec_client.cc

@@ -36,8 +36,14 @@ CodecClient::CodecClient(Type type, Network::ClientConnectionPtr&& connection,
   connection_->addConnectionCallbacks(*this);
   connection_->addReadFilter(Network::ReadFilterSharedPtr{new CodecReadFilter(*this)});
 
-  ENVOY_CONN_LOG(debug, "connecting", *connection_);
-  connection_->connect();
+  // In general, codecs are handed new not-yet-connected connections, but in the
+  // case of ALPN, the codec may be handed an already connected connection.
+  if (!connection_->connecting()) {
+    connected_ = true;
+  } else {
+    ENVOY_CONN_LOG(debug, "connecting", *connection_);
+    connection_->connect();
+  }
 
   if (idle_timeout_) {
     idle_timer_ = dispatcher.createTimer([this]() -> void { onIdleTimeout(); });

source/common/http/conn_pool_base.h

@@ -92,6 +92,13 @@ class ActiveClient : public Envoy::ConnectionPool::ActiveClient {
     initialize(data, parent);
   }
 
+  ActiveClient(HttpConnPoolImplBase& parent, uint64_t lifetime_stream_limit,
+               uint64_t concurrent_stream_limit, Upstream::Host::CreateConnectionData& data)
+      : Envoy::ConnectionPool::ActiveClient(parent, lifetime_stream_limit,
+                                            concurrent_stream_limit) {
+    initialize(data, parent);
+  }
+
   void initialize(Upstream::Host::CreateConnectionData& data, HttpConnPoolImplBase& parent) {
     real_host_description_ = data.host_description_;
     codec_client_ = parent.createCodecClient(data);
@@ -104,6 +111,7 @@ class ActiveClient : public Envoy::ConnectionPool::ActiveClient {
          &parent_.host()->cluster().stats().bind_errors_, nullptr});
   }
 
+  absl::optional<Http::Protocol> protocol() const override { return codec_client_->protocol(); }
   void close() override { codec_client_->close(); }
   virtual Http::RequestEncoder& newStreamEncoder(Http::ResponseDecoder& response_decoder) PURE;
   void onEvent(Network::ConnectionEvent event) override {

source/common/http/http1/conn_pool.cc

@@ -46,7 +46,7 @@ ConnPoolImpl::StreamWrapper::~StreamWrapper() {
   // Upstream connection might be closed right after response is complete. Setting delay=true
   // here to attach pending requests in next dispatcher loop to handle that case.
   // https://github.com/envoyproxy/envoy/issues/2715
-  parent_.parent().onStreamClosed(parent_, true);
+  parent_.parent_.onStreamClosed(parent_, true);
 }
 
 void ConnPoolImpl::StreamWrapper::onEncodeComplete() { encode_complete_ = true; }
@@ -97,12 +97,21 @@ void ConnPoolImpl::StreamWrapper::onDecodeComplete() {
   }
 }
 
-ConnPoolImpl::ActiveClient::ActiveClient(ConnPoolImpl& parent)
+ConnPoolImpl::ActiveClient::ActiveClient(HttpConnPoolImplBase& parent)
     : Envoy::Http::ActiveClient(
-          parent, parent.host_->cluster().maxRequestsPerConnection(),
+          parent, parent.host()->cluster().maxRequestsPerConnection(),
           1 // HTTP1 always has a concurrent-request-limit of 1 per connection.
       ) {
-  parent.host_->cluster().stats().upstream_cx_http1_total_.inc();
+  parent.host()->cluster().stats().upstream_cx_http1_total_.inc();
+}
+
+ConnPoolImpl::ActiveClient::ActiveClient(HttpConnPoolImplBase& parent,
+                                         Upstream::Host::CreateConnectionData& data)
+    : Envoy::Http::ActiveClient(
+          parent, parent.host()->cluster().maxRequestsPerConnection(),
+          1, // HTTP1 always has a concurrent-request-limit of 1 per connection.
+          data) {
+  parent.host()->cluster().stats().upstream_cx_http1_total_.inc();
 }
 
 bool ConnPoolImpl::ActiveClient::closingWithIncompleteStream() const {

source/common/http/http1/conn_pool.h

@@ -29,7 +29,6 @@ class ConnPoolImpl : public Http::HttpConnPoolImplBase {
   // ConnPoolImplBase
   Envoy::ConnectionPool::ActiveClientPtr instantiateActiveClient() override;
 
-protected:
   class ActiveClient;
 
   struct StreamWrapper : public RequestEncoderWrapper,
@@ -63,7 +62,8 @@ class ConnPoolImpl : public Http::HttpConnPoolImplBase {
 
   class ActiveClient : public Envoy::Http::ActiveClient {
   public:
-    ActiveClient(ConnPoolImpl& parent);
+    ActiveClient(HttpConnPoolImplBase& parent);
+    ActiveClient(HttpConnPoolImplBase& parent, Upstream::Host::CreateConnectionData& data);
 
     ConnPoolImpl& parent() { return *static_cast<ConnPoolImpl*>(&parent_); }
 

source/common/http/http2/conn_pool.cc

@@ -77,6 +77,16 @@ ConnPoolImpl::ActiveClient::ActiveClient(Envoy::Http::HttpConnPoolImplBase& pare
   parent.host()->cluster().stats().upstream_cx_http2_total_.inc();
 }
 
+ConnPoolImpl::ActiveClient::ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
+                                         Upstream::Host::CreateConnectionData& data)
+    : Envoy::Http::ActiveClient(
+          parent, maxStreamsPerConnection(parent.host()->cluster().maxRequestsPerConnection()),
+          parent.host()->cluster().http2Options().max_concurrent_streams().value(), data) {
+  codec_client_->setCodecClientCallbacks(*this);
+  codec_client_->setCodecConnectionCallbacks(*this);
+  parent.host()->cluster().stats().upstream_cx_http2_total_.inc();
+}
+
 bool ConnPoolImpl::ActiveClient::closingWithIncompleteStream() const {
   return closed_with_active_rq_;
 }

source/common/http/http2/conn_pool.h

@@ -37,7 +37,7 @@ class ConnPoolImpl : public Envoy::Http::HttpConnPoolImplBase {
                  Upstream::Host::CreateConnectionData& data);
     ~ActiveClient() override = default;
 
-    ConnPoolImpl& parent() { return static_cast<ConnPoolImpl&>(parent_); }
+    HttpConnPoolImplBase& parent() { return static_cast<HttpConnPoolImplBase&>(parent_); }
 
     // ConnPoolImpl::ActiveClient
     bool closingWithIncompleteStream() const override;

source/common/http/mixed_conn_pool.cc

@@ -0,0 +1,65 @@
+#include "common/http/mixed_conn_pool.h"
+
+#include "common/http/codec_client.h"
+#include "common/http/http1/conn_pool.h"
+#include "common/http/http2/conn_pool.h"
+#include "common/http/utility.h"
+#include "common/tcp/conn_pool.h"
+
+namespace Envoy {
+namespace Http {
+
+Envoy::ConnectionPool::ActiveClientPtr HttpConnPoolImplMixed::instantiateActiveClient() {
+  return std::make_unique<Tcp::ActiveTcpClient>(*this,
+                                                Envoy::ConnectionPool::ConnPoolImplBase::host(), 1);
+}
+
+CodecClientPtr
+HttpConnPoolImplMixed::createCodecClient(Upstream::Host::CreateConnectionData& data) {
+  auto protocol =
+      protocol_ == Protocol::Http11 ? CodecClient::Type::HTTP1 : CodecClient::Type::HTTP2;
+  CodecClientPtr codec{new CodecClientProd(protocol, std::move(data.connection_),
+                                           data.host_description_, dispatcher_, random_generator_)};
+  return codec;
+}
+
+void HttpConnPoolImplMixed::onConnected(Envoy::ConnectionPool::ActiveClient& client) {
+  // When we upgrade from a TCP client to non-TCP we get a spurious onConnected
+  // from the new client. Ignore that.
+  if (client.protocol() != absl::nullopt) {
+    return;
+  }
+
+  connected_ = true;
+  // If an old TLS stack does not negotiate alpn, it likely does not support
+  // HTTP/2. Fail over to HTTP/1.
+  protocol_ = Protocol::Http11;
+  auto tcp_client = static_cast<Tcp::ActiveTcpClient*>(&client);
+  std::string alpn = tcp_client->connection_->nextProtocol();
+  if (!alpn.empty()) {
+    if (alpn == Http::Utility::AlpnNames::get().Http11) {
+      protocol_ = Http::Protocol::Http11;
+    } else if (alpn == Http::Utility::AlpnNames::get().Http2) {
+      protocol_ = Http::Protocol::Http2;
+    }
+  }
+
+  Upstream::Host::CreateConnectionData data{std::move(tcp_client->connection_),
+                                            client.real_host_description_};
+  data.connection_->removeConnectionCallbacks(*tcp_client);
+  data.connection_->removeReadFilter(tcp_client->read_filter_handle_);
+  dispatcher_.deferredDelete(client.removeFromList(owningList(client.state_)));
+
+  std::unique_ptr<ActiveClient> new_client;
+  if (protocol_ == Http::Protocol::Http11) {
+    new_client = std::make_unique<Http1::ConnPoolImpl::ActiveClient>(*this, data);
+  } else {
+    new_client = std::make_unique<Http2::ConnPoolImpl::ActiveClient>(*this, data);
+  }
+  connecting_stream_capacity_ += new_client->effectiveConcurrentStreamLimit();
+  new_client->state_ = ActiveClient::State::CONNECTING;
+  LinkedList::moveIntoList(std::move(new_client), owningList(new_client->state_));
+}
+
+} // namespace Http
+} // namespace Envoy

source/common/http/mixed_conn_pool.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "common/http/conn_pool_base.h"
+
+namespace Envoy {
+namespace Http {
+
+// An HTTP connection pool which supports both HTTP/1 and HTTP/2 based on ALPN
+class HttpConnPoolImplMixed : public HttpConnPoolImplBase {
+public:
+  HttpConnPoolImplMixed(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator,
+                        Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,
+                        const Network::ConnectionSocket::OptionsSharedPtr& options,
+                        const Network::TransportSocketOptionsSharedPtr& transport_socket_options)
+      : HttpConnPoolImplBase(std::move(host), std::move(priority), dispatcher, options,
+                             transport_socket_options, random_generator,
+                             {Protocol::Http2, Protocol::Http11}) {}
+
+  Http::Protocol protocol() const override {
+    // This is a pure debug check to ensure call sites defer protocol() calls
+    // until ALPN has a chance to be negotiated.
+    ASSERT(connected_);
+    return protocol_;
+  }
+  Envoy::ConnectionPool::ActiveClientPtr instantiateActiveClient() override;
+  CodecClientPtr createCodecClient(Upstream::Host::CreateConnectionData& data) override;
+
+  void onConnected(Envoy::ConnectionPool::ActiveClient& client) override;
+
+private:
+  bool connected_{};
+  // Default to HTTP/1, as servers which don't support ALPN are probably HTTP/1 only.
+  Http::Protocol protocol_ = Protocol::Http11;
+};
+
+} // namespace Http
+} // namespace Envoy