http2: limit the number of outbound frames (#23)

Limit the number of outbound (these, waiting to be written into the socket)
HTTP/2 frames. When the limit is exceeded the connection is terminated.

This mitigates flood exploits where a client continually sends frames that
are not subject to flow control without reading server responses.

Fixes CVE-2019-9512, CVE-2019-9514 and CVE-2019-9515.

Signed-off-by: Yan Avlasov <[email protected]>

api/envoy/api/v2/core/protocol.proto

@@ -91,6 +91,21 @@ message Http2ProtocolOptions {
   // docs](https://github.com/envoyproxy/envoy/blob/master/source/docs/h2_metadata.md) for more
   // information.
   bool allow_metadata = 6;
+
+  // Limit the number of pending outbound downstream frames of all types (frames that are waiting to
+  // be written into the socket). Exceeding this limit triggers flood mitigation and connection is
+  // terminated. The "http2.outbound_flood" stat tracks the number of terminated connections due to
+  // flood mitigation. The default limit is 10000.
+  // [#comment:TODO: implement same limits for upstream outbound frames as well.]
+  google.protobuf.UInt32Value max_outbound_frames = 7 [(validate.rules).uint32 = {gte: 1}];
+
+  // Limit the number of pending outbound downstream frames of types PING, SETTINGS and RST_STREAM,
+  // preventing high memory utilization when receiving continuous stream of these frames. Exceeding
+  // this limit triggers flood mitigation and connection is terminated. The
+  // "http2.outbound_control_flood" stat tracks the number of terminated connections due to flood
+  // mitigation. The default limit is 1000.
+  // [#comment:TODO: implement same limits for upstream outbound frames as well.]
+  google.protobuf.UInt32Value max_outbound_control_frames = 8 [(validate.rules).uint32 = {gte: 1}];
 }
 
 // [#not-implemented-hide:]

docs/root/configuration/http_conn_man/stats.rst

@@ -122,6 +122,8 @@ All http2 statistics are rooted at *http2.*
 
    header_overflow, Counter, Total number of connections reset due to the headers being larger than the :ref:`configured value <envoy_api_field_config.filter.network.http_connection_manager.v2.HttpConnectionManager.max_request_headers_kb>`.
    headers_cb_no_stream, Counter, Total number of errors where a header callback is called without an associated stream. This tracks an unexpected occurrence due to an as yet undiagnosed bug
+   outbound_flood, Counter, Total number of connections terminated for exceeding the limit on outbound frames of all types. The limit is configured by setting the :ref:`max_outbound_frames config setting <envoy_api_field_core.Http2ProtocolOptions.max_outbound_frames>`.
+   outbound_control_flood, Counter, "Total number of connections terminated for exceeding the limit on outbound frames of types PING, SETTINGS and RST_STREAM. The limit is configured by setting the :ref:`max_outbound_control_frames config setting <envoy_api_field_core.Http2ProtocolOptions.max_outbound_control_frames>`."
    rx_messaging_error, Counter, Total number of invalid received frames that violated `section 8 <https://tools.ietf.org/html/rfc7540#section-8>`_ of the HTTP/2 spec. This will result in a *tx_reset*
    rx_reset, Counter, Total number of reset stream frames received by Envoy
    too_many_header_frames, Counter, Total number of times an HTTP2 connection is reset due to receiving too many headers frames. Envoy currently supports proxying at most one header frame for 100-Continue one non-100 response code header frame and one frame with trailers

docs/root/intro/version_history.rst

@@ -26,6 +26,12 @@ Version history
 * upstream: added network filter chains to upstream connections, see :ref:`filters<envoy_api_field_Cluster.filters>`.
 * zookeeper: parse responses and emit latency stats.
 
+1.11.1 (August 13, 2019)
+========================
+* http: added mitigation of client initiated atacks that result in flooding of the outbound queue of downstream HTTP/2 connections.
+* http: added :ref:`outbound_flood <config_http_conn_man_stats_per_codec>` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit. The limit is configured by setting the :ref:`max_outbound_frames config setting <envoy_api_field_core.Http2ProtocolOptions.max_outbound_frames>`
+* http: added :ref:`outbound_control_flood <config_http_conn_man_stats_per_codec>` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit for PING, SETTINGS and RST_STREAM frames. The limit is configured by setting the :ref:`max_outbound_control_frames config setting <envoy_api_field_core.Http2ProtocolOptions.max_outbound_control_frames>`.
+
 1.11.0 (July 11, 2019)
 ======================
 * access log: added a new field for downstream TLS session ID to file and gRPC access logger.

include/envoy/buffer/buffer.h

@@ -33,6 +33,7 @@ struct RawSlice {
  */
 class BufferFragment {
 public:
+  virtual ~BufferFragment() = default;
   /**
    * @return const void* a pointer to the referenced data.
    */
@@ -47,9 +48,6 @@ class BufferFragment {
    * Called by a buffer when the referenced data is no longer needed.
    */
   virtual void done() PURE;
-
-protected:
-  virtual ~BufferFragment() = default;
 };
 
 /**

include/envoy/http/codec.h

@@ -235,6 +235,8 @@ struct Http2Settings {
   uint32_t initial_connection_window_size_{DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE};
   bool allow_connect_{DEFAULT_ALLOW_CONNECT};
   bool allow_metadata_{DEFAULT_ALLOW_METADATA};
+  uint32_t max_outbound_frames_{DEFAULT_MAX_OUTBOUND_FRAMES};
+  uint32_t max_outbound_control_frames_{DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES};
 
   // disable HPACK compression
   static const uint32_t MIN_HPACK_TABLE_SIZE = 0;
@@ -272,6 +274,11 @@ struct Http2Settings {
   static const bool DEFAULT_ALLOW_CONNECT = false;
   // By default Envoy does not allow METADATA support.
   static const bool DEFAULT_ALLOW_METADATA = false;
+
+  // Default limit on the number of outbound frames of all types.
+  static const uint32_t DEFAULT_MAX_OUTBOUND_FRAMES = 10000;
+  // Default limit on the number of outbound frames of types PING, SETTINGS and RST_STREAM.
+  static const uint32_t DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES = 1000;
 };
 
 /**

source/common/buffer/buffer_impl.h

@@ -193,7 +193,8 @@ class Slice {
 
 using SlicePtr = std::unique_ptr<Slice>;
 
-class OwnedSlice : public Slice, public InlineStorage {
+// OwnedSlice can not be derived from as it has variable sized array as member.
+class OwnedSlice final : public Slice, public InlineStorage {
 public:
   /**
    * Create an empty OwnedSlice.
@@ -563,5 +564,46 @@ class OwnedImpl : public LibEventInstance {
   Event::Libevent::BufferPtr buffer_;
 };
 
+using BufferFragmentPtr = std::unique_ptr<BufferFragment>;
+
+/**
+ * An implementation of BufferFragment where a releasor callback is called when the data is
+ * no longer needed. Copies data into internal buffer.
+ */
+class OwnedBufferFragmentImpl final : public BufferFragment, public InlineStorage {
+public:
+  using Releasor = std::function<void(const OwnedBufferFragmentImpl*)>;
+
+  /**
+   * Copies the data into internal buffer. The releasor is called when the data has been
+   * fully drained or the buffer that contains this fragment is destroyed.
+   * @param data external data to reference
+   * @param releasor a callback function to be called when data is no longer needed.
+   */
+
+  static BufferFragmentPtr create(absl::string_view data, const Releasor& releasor) {
+    return BufferFragmentPtr(new (sizeof(OwnedBufferFragmentImpl) + data.size())
+                                 OwnedBufferFragmentImpl(data, releasor));
+  }
+
+  // Buffer::BufferFragment
+  const void* data() const override { return data_; }
+  size_t size() const override { return size_; }
+  void done() override { releasor_(this); }
+
+private:
+  OwnedBufferFragmentImpl(absl::string_view data, const Releasor& releasor)
+      : releasor_(releasor), size_(data.size()) {
+    ASSERT(releasor != nullptr);
+    memcpy(data_, data.data(), data.size());
+  }
+
+  const Releasor releasor_;
+  const size_t size_;
+  uint8_t data_[];
+};
+
+using OwnedBufferFragmentImplPtr = std::unique_ptr<OwnedBufferFragmentImpl>;
+
 } // namespace Buffer
 } // namespace Envoy

source/common/http/conn_manager_impl.cc

@@ -258,6 +258,19 @@ StreamDecoder& ConnectionManagerImpl::newStream(StreamEncoder& response_encoder,
   return **streams_.begin();
 }
 
+void ConnectionManagerImpl::handleCodecException(const char* error,
+                                                 Network::ConnectionCloseType close_type) {
+  ENVOY_CONN_LOG(debug, "dispatch error: {}", read_callbacks_->connection(), error);
+
+  // In the protocol error case, we need to reset all streams now. If the close_type is
+  // FlushWriteAndDelay, the connection might stick around long enough for a pending stream to come
+  // back and try to encode. In other cases it avoids needless processing of upstream responses when
+  // downstream connection is closed.
+  resetAllStreams();
+
+  read_callbacks_->connection().close(close_type);
+}
+
 Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool) {
   if (!codec_) {
     codec_ = config_.createCodec(read_callbacks_->connection(), data, *this);
@@ -276,18 +289,15 @@ Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool
 
     try {
       codec_->dispatch(data);
+    } catch (const FrameFloodException& e) {
+      // Abortively close flooded connections
+      handleCodecException(e.what(), Network::ConnectionCloseType::NoFlush);
+      return Network::FilterStatus::StopIteration;
     } catch (const CodecProtocolException& e) {
+      stats_.named_.downstream_cx_protocol_error_.inc();
       // HTTP/1.1 codec has already sent a 400 response if possible. HTTP/2 codec has already sent
       // GOAWAY.
-      ENVOY_CONN_LOG(debug, "dispatch error: {}", read_callbacks_->connection(), e.what());
-      stats_.named_.downstream_cx_protocol_error_.inc();
-
-      // In the protocol error case, we need to reset all streams now. Since we do a flush write and
-      // delayed close, the connection might stick around long enough for a pending stream to come
-      // back and try to encode.
-      resetAllStreams();
-
-      read_callbacks_->connection().close(Network::ConnectionCloseType::FlushWriteAndDelay);
+      handleCodecException(e.what(), Network::ConnectionCloseType::FlushWriteAndDelay);
       return Network::FilterStatus::StopIteration;
     }
 

source/common/http/conn_manager_impl.h

@@ -653,6 +653,7 @@ class ConnectionManagerImpl : Logger::Loggable<Logger::Id::http>,
   void onDrainTimeout();
   void startDrainSequence();
   Tracing::HttpTracer& tracer() { return http_context_.tracer(); }
+  void handleCodecException(const char* error, Network::ConnectionCloseType close_type);
 
   enum class DrainState { NotDraining, Draining, Closing };
 

source/common/http/exception.h

@@ -16,6 +16,14 @@ class CodecProtocolException : public EnvoyException {
   CodecProtocolException(const std::string& message) : EnvoyException(message) {}
 };
 
+/**
+ * Raised when outbound frame queue flood is detected.
+ */
+class FrameFloodException : public CodecProtocolException {
+public:
+  FrameFloodException(const std::string& message) : CodecProtocolException(message) {}
+};
+
 /**
  * Raised when a response is received on a connection that did not send a request. In practice
  * this can only happen on HTTP/1.1 connections.

source/common/http/http2/codec_impl.cc

@@ -11,6 +11,7 @@
 #include "envoy/stats/scope.h"
 
 #include "common/common/assert.h"
+#include "common/common/cleanup.h"
 #include "common/common/enum_to_int.h"
 #include "common/common/fmt.h"
 #include "common/common/stack_array.h"
@@ -251,7 +252,13 @@ int ConnectionImpl::StreamImpl::onDataSourceSend(const uint8_t* framehd, size_t
   // https://nghttp2.org/documentation/types.html#c.nghttp2_send_data_callback
   static const uint64_t FRAME_HEADER_SIZE = 9;
 
-  Buffer::OwnedImpl output(framehd, FRAME_HEADER_SIZE);
+  Buffer::OwnedImpl output;
+  if (!parent_.addOutboundFrameFragment(output, framehd, FRAME_HEADER_SIZE)) {
+    ENVOY_CONN_LOG(debug, "error sending data frame: Too many frames in the outbound queue",
+                   parent_.connection_);
+    return NGHTTP2_ERR_FLOODED;
+  }
+
   output.move(pending_send_data_, length);
   parent_.connection_.write(output, false);
   return 0;
@@ -348,6 +355,10 @@ void ConnectionImpl::dispatch(Buffer::Instance& data) {
     dispatching_ = true;
     ssize_t rc =
         nghttp2_session_mem_recv(session_, static_cast<const uint8_t*>(slice.mem_), slice.len_);
+    if (rc == NGHTTP2_ERR_FLOODED) {
+      throw FrameFloodException(
+          "Flooding was detected in this HTTP/2 session, and it must be closed");
+    }
     if (rc != static_cast<ssize_t>(slice.len_)) {
       throw CodecProtocolException(fmt::format("{}", nghttp2_strerror(rc)));
     }
@@ -555,9 +566,77 @@ int ConnectionImpl::onInvalidFrame(int32_t stream_id, int error_code) {
   return NGHTTP2_ERR_CALLBACK_FAILURE;
 }
 
+int ConnectionImpl::onBeforeFrameSend(const nghttp2_frame* frame) {
+  ENVOY_CONN_LOG(trace, "about to sent frame type={}, flags={}", connection_,
+                 static_cast<uint64_t>(frame->hd.type), static_cast<uint64_t>(frame->hd.flags));
+  ASSERT(!is_outbound_flood_monitored_control_frame_);
+  // Flag flood monitored outbound control frames.
+  is_outbound_flood_monitored_control_frame_ =
+      ((frame->hd.type == NGHTTP2_PING || frame->hd.type == NGHTTP2_SETTINGS) &&
+       frame->hd.flags & NGHTTP2_FLAG_ACK) ||
+      frame->hd.type == NGHTTP2_RST_STREAM;
+  return 0;
+}
+
+void ConnectionImpl::incrementOutboundFrameCount(bool is_outbound_flood_monitored_control_frame) {
+  ++outbound_frames_;
+  if (is_outbound_flood_monitored_control_frame) {
+    ++outbound_control_frames_;
+  }
+  checkOutboundQueueLimits();
+}
+
+bool ConnectionImpl::addOutboundFrameFragment(Buffer::OwnedImpl& output, const uint8_t* data,
+                                              size_t length) {
+  // Reset the outbound frame type (set in the onBeforeFrameSend callback) since the
+  // onBeforeFrameSend callback is not called for DATA frames.
+  bool is_outbound_flood_monitored_control_frame = false;
+  std::swap(is_outbound_flood_monitored_control_frame, is_outbound_flood_monitored_control_frame_);
+  try {
+    incrementOutboundFrameCount(is_outbound_flood_monitored_control_frame);
+  } catch (const FrameFloodException&) {
+    return false;
+  }
+
+  auto fragment = Buffer::OwnedBufferFragmentImpl::create(
+      absl::string_view(reinterpret_cast<const char*>(data), length),
+      is_outbound_flood_monitored_control_frame ? control_frame_buffer_releasor_
+                                                : frame_buffer_releasor_);
+
+  // The Buffer::OwnedBufferFragmentImpl object will be deleted in the *frame_buffer_releasor_
+  // callback.
+  output.addBufferFragment(*fragment.release());
+  return true;
+}
+
+void ConnectionImpl::releaseOutboundFrame(const Buffer::OwnedBufferFragmentImpl* fragment) {
+  ASSERT(outbound_frames_ >= 1);
+  --outbound_frames_;
+  delete fragment;
+}
+
+void ConnectionImpl::releaseOutboundControlFrame(const Buffer::OwnedBufferFragmentImpl* fragment) {
+  ASSERT(outbound_control_frames_ >= 1);
+  --outbound_control_frames_;
+  releaseOutboundFrame(fragment);
+}
+
 ssize_t ConnectionImpl::onSend(const uint8_t* data, size_t length) {
   ENVOY_CONN_LOG(trace, "send data: bytes={}", connection_, length);
-  Buffer::OwnedImpl buffer(data, length);
+  Buffer::OwnedImpl buffer;
+  if (!addOutboundFrameFragment(buffer, data, length)) {
+    ENVOY_CONN_LOG(debug, "error sending frame: Too many frames in the outbound queue.",
+                   connection_);
+    return NGHTTP2_ERR_FLOODED;
+  }
+
+  // While the buffer is transient the fragment it contains will be moved into the
+  // write_buffer_ of the underlying connection_ by the write method below.
+  // This creates lifetime dependency between the write_buffer_ of the underlying connection
+  // and the codec object. Specifically the write_buffer_ MUST be either fully drained or
+  // deleted before the codec object is deleted. This is presently guaranteed by the
+  // destruction order of the Network::ConnectionImpl object where write_buffer_ is
+  // destroyed before the filter_manager_ which owns the codec through Http::ConnectionManagerImpl.
   connection_.write(buffer, false);
   return length;
 }
@@ -663,6 +742,15 @@ void ConnectionImpl::sendPendingFrames() {
   int rc = nghttp2_session_send(session_);
   if (rc != 0) {
     ASSERT(rc == NGHTTP2_ERR_CALLBACK_FAILURE);
+    // For errors caused by the pending outbound frame flood the FrameFloodException has
+    // to be thrown. However the nghttp2 library returns only the generic error code for
+    // all failure types. Check queue limits and throw FrameFloodException if they were
+    // exceeded.
+    if (outbound_frames_ > max_outbound_frames_ ||
+        outbound_control_frames_ > max_outbound_control_frames_) {
+      throw FrameFloodException("Too many frames in the outbound queue.");
+    }
+
     throw CodecProtocolException(fmt::format("{}", nghttp2_strerror(rc)));
   }
 
@@ -810,6 +898,11 @@ ConnectionImpl::Http2Callbacks::Http2Callbacks() {
         return static_cast<ConnectionImpl*>(user_data)->onFrameSend(frame);
       });
 
+  nghttp2_session_callbacks_set_before_frame_send_callback(
+      callbacks_, [](nghttp2_session*, const nghttp2_frame* frame, void* user_data) -> int {
+        return static_cast<ConnectionImpl*>(user_data)->onBeforeFrameSend(frame);
+      });
+
   nghttp2_session_callbacks_set_on_frame_not_send_callback(
       callbacks_, [](nghttp2_session*, const nghttp2_frame*, int, void*) -> int {
         // We used to always return failure here but it looks now this can get called if the other
@@ -979,6 +1072,31 @@ int ServerConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& na
   return saveHeader(frame, std::move(name), std::move(value));
 }
 
+void ServerConnectionImpl::checkOutboundQueueLimits() {
+  if (outbound_frames_ > max_outbound_frames_ && dispatching_downstream_data_) {
+    stats_.outbound_flood_.inc();
+    throw FrameFloodException("Too many frames in the outbound queue.");
+  }
+  if (outbound_control_frames_ > max_outbound_control_frames_ && dispatching_downstream_data_) {
+    stats_.outbound_control_flood_.inc();
+    throw FrameFloodException("Too many control frames in the outbound queue.");
+  }
+}
+
+void ServerConnectionImpl::dispatch(Buffer::Instance& data) {
+  ASSERT(!dispatching_downstream_data_);
+  dispatching_downstream_data_ = true;
+
+  // Make sure the dispatching_downstream_data_ is set to false even
+  // when ConnectionImpl::dispatch throws an exception.
+  Cleanup cleanup([this]() { dispatching_downstream_data_ = false; });
+
+  // Make sure downstream outbound queue was not flooded by the upstream frames.
+  checkOutboundQueueLimits();
+
+  ConnectionImpl::dispatch(data);
+}
+
 } // namespace Http2
 } // namespace Http
 } // namespace Envoy