Merge pull request maistra#7 from dmitri-d/maistra-1.1-upstream-relea…

…se-1.4.6-patch

Fixes MAISTRA-1265: Upstream release 1.4.6 + alpn fixes + latest bssl_wrapper

bazel/external/openssl_includes-1.patch

@@ -0,0 +1,13 @@
+diff --git a/ssl/packet_locl.h b/ssl/packet_locl.h
+index 860360b8b2..49c719285f 100644
+--- a/ssl/packet_locl.h
++++ b/ssl/packet_locl.h
+@@ -426,7 +426,7 @@ __owur static ossl_inline int PACKET_memdup(const PACKET *pkt,
+     if (length == 0)
+         return 1;
+
+-    *data = OPENSSL_memdup(pkt->curr, length);
++    *data = (unsigned char *)OPENSSL_memdup(pkt->curr, length);
+     if (*data == NULL)
+         return 0;
+

bazel/external/openssl_includes.BUILD

@@ -0,0 +1,18 @@
+cc_library(
+    name = "openssl_includes_lib",
+    copts = ["-Wno-error=error"],
+    hdrs = [
+            "e_os.h", 
+            "ssl/ssl_locl.h", 
+            "ssl/packet_locl.h", 
+            "ssl/record/record.h", 
+            "ssl/statem/statem.h", 
+            "include/internal/dane.h",
+            "include/internal/nelem.h",
+            "include/internal/numbers.h",
+            "include/internal/refcount.h",
+            "include/internal/tsan_assist.h",
+    ],
+    includes = ["ssl", "ssl/record", "ssl/statem", "include",],
+    visibility = ["//visibility:public"],
+)

bazel/repositories.bzl

@@ -109,6 +109,7 @@ def envoy_dependencies(skip_targets = []):
 
     # EXTERNAL OPENSSL
     _openssl()
+    _openssl_includes()
     _bssl_wrapper()
     _openssl_cbs()
 
@@ -182,6 +183,21 @@ def _openssl():
         actual = "@openssl//:openssl-lib",
 )
 
+def _openssl_includes():
+    _repository_impl(
+        name = "com_github_openssl_openssl",
+        build_file = "@envoy//bazel/external:openssl_includes.BUILD",
+        patches = [
+            "@envoy//bazel/external:openssl_includes-1.patch",
+        ],
+        patch_args = ["-p1"],
+    )
+    native.bind(
+        name = "openssl_includes_lib",
+        actual = "@com_github_openssl_openssl//:openssl_includes_lib",
+)
+
+
 #EXTERNAL OPENSSL
 def _bssl_wrapper():
     _repository_impl("bssl_wrapper")

bazel/repository_locations.bzl

@@ -21,11 +21,16 @@ REPOSITORY_LOCATIONS = dict(
         strip_prefix = "envoy-build-tools-a6b28555badcb18d6be924c8fc1bea49971656b8",
         urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/a6b28555badcb18d6be924c8fc1bea49971656b8.tar.gz"],
     ),
+    com_github_openssl_openssl = dict(
+        sha256 = "cf26f056a955cff721d3a3c08d8126d1e4f69803e08c9600dac3b6b7158586d6",
+        strip_prefix = "openssl-894da2fb7ed5d314ee5c2fc9fd2d9b8b74111596",
+        urls = ["https://github.com/openssl/openssl/archive/894da2fb7ed5d314ee5c2fc9fd2d9b8b74111596.tar.gz"],
+     ),
     #EXTERNAL OPENSSL
     bssl_wrapper = dict(
-        sha256 = "81a59d013096015a93269325ee4148d826ffd7a9f019f622850a2b86974b9748",
-        strip_prefix = "bssl_wrapper-2eaed8832e12a0fada8f08a5e23522e035b80784",
-        urls = ["https://github.com/maistra/bssl_wrapper/archive/2eaed8832e12a0fada8f08a5e23522e035b80784.tar.gz"],
+        sha256 = "d84ea7d190210145695e5b172e8e6fb23f3464360da5efab5a1ae1a973c21f57",
+        strip_prefix = "bssl_wrapper-c9649facde3ab1d8bc871c7375a8946c50950e97",
+        urls = ["https://github.com/maistra/bssl_wrapper/archive/c9649facde3ab1d8bc871c7375a8946c50950e97.tar.gz"],
     ),
     #EXTERNAL OPENSSL
     openssl_cbs = dict(

docs/root/configuration/http/http_conn_man/stats.rst

@@ -111,6 +111,7 @@ All http1 statistics are rooted at *http1.*
    :widths: 1, 1, 2
 
    metadata_not_supported_error, Counter, Total number of metadata dropped during HTTP/1 encoding
+   response_flood, Counter, Total number of connections closed due to response flooding
 
 Http2 codec statistics
 ~~~~~~~~~~~~~~~~~~~~~~

docs/root/intro/arch_overview/security/rbac_filter.rst

@@ -81,6 +81,7 @@ The following attributes are exposed to the language runtime:
    response.headers, string map, All response headers
    response.trailers, string map, All response trailers
    response.size, int, Size of the response body
+   response.total_size, int, Total size of the response including the approximate uncompressed size of the headers and the trailers
    response.flags, int, Additional details about the response beyond the standard response code
    source.address, string, Downstream connection remote address
    source.port, int, Downstream connection remote port

docs/root/intro/version_history.rst

@@ -1,6 +1,25 @@
 Version history
 ---------------
 
+1.12.3 (Pending)
+==========================
+* buffer: force copy when appending small slices to OwnedImpl buffer to avoid fragmentation.
+* listeners: fixed issue where :ref:`TLS inspector listener filter <config_listener_filters_tls_inspector>` could have been bypassed by a client using only TLS 1.3.
+* sds: fixed the SDS vulnerability that TLS validation context (e.g., subject alt name or hash) cannot be effectively validated in some cases.
+* http: added HTTP/1.1 flood protection. Can be temporarily disabled using the runtime feature `envoy.reloadable_features.http1_flood_protection`.
+
+1.12.2 (December 10, 2019)
+==========================
+* http: fixed CVE-2019-18801 by allocating sufficient memory for request headers.
+* http: fixed CVE-2019-18802 by implementing stricter validation of HTTP/1 headers.
+* http: trim LWS at the end of header keys, for correct HTTP/1.1 header parsing.
+* http: added strict authority checking. This can be reversed temporarily by setting the runtime feature `envoy.reloadable_features.strict_authority_validation` to false.
+* route config: fixed CVE-2019-18838 by checking for presence of host/path headers.
+
+1.12.1 (November 8, 2019)
+=========================
+* listener: fixed CVE-2019-18836 by clearing accept filters before connection creation.
+
 1.12.0 (October 31, 2019)
 =========================
 * access log: added a new flag for :ref:`downstream protocol error <envoy_api_field_data.accesslog.v2.ResponseFlags.downstream_protocol_error>`.

include/envoy/buffer/buffer.h

@@ -101,8 +101,8 @@ class Instance {
   /**
    * Commit a set of slices originally obtained from reserve(). The number of slices should match
    * the number obtained from reserve(). The size of each slice can also be altered. Commit must
-   * occur following a reserve() without any mutating operations in between other than to the iovecs
-   * len_ fields.
+   * occur once following a reserve() without any mutating operations in between other than to the
+   * iovecs len_ fields.
    * @param iovecs supplies the array of slices to commit.
    * @param num_iovecs supplies the size of the slices array.
    */

source/common/buffer/buffer_impl.cc

@@ -10,8 +10,15 @@
 
 namespace Envoy {
 namespace Buffer {
-
-void OwnedImpl::add(const void* data, uint64_t size) {
+namespace {
+// This size has been determined to be optimal from running the
+// //test/integration:http_benchmark benchmark tests.
+// TODO(yanavlasov): This may not be optimal for all hardware configurations or traffic patterns and
+// may need to be configurable in the future.
+constexpr uint64_t CopyThreshold = 512;
+} // namespace
+
+void OwnedImpl::addImpl(const void* data, uint64_t size) {
   if (old_impl_) {
     evbuffer_add(buffer_.get(), data, size);
   } else {
@@ -30,6 +37,8 @@ void OwnedImpl::add(const void* data, uint64_t size) {
   }
 }
 
+void OwnedImpl::add(const void* data, uint64_t size) { addImpl(data, size); }
+
 void OwnedImpl::addBufferFragment(BufferFragment& fragment) {
   if (old_impl_) {
     evbuffer_add_reference(
@@ -150,6 +159,11 @@ void OwnedImpl::commit(RawSlice* iovecs, uint64_t num_iovecs) {
       }
     }
 
+    // In case an extra slice was reserved, remove empty slices from the end of the buffer.
+    while (!slices_.empty() && slices_.back()->dataSize() == 0) {
+      slices_.pop_back();
+    }
+
     ASSERT(num_slices_committed > 0);
   }
 }
@@ -213,6 +227,11 @@ void OwnedImpl::drain(uint64_t size) {
       }
     }
   }
+  // Make sure to drain any zero byte fragments that might have been added as
+  // sentinels for flushed data.
+  while (!slices_.empty() && slices_.front()->dataSize() == 0) {
+    slices_.pop_front();
+  }
 }
 
 uint64_t OwnedImpl::getRawSlices(RawSlice* out, uint64_t out_size) const {
@@ -299,6 +318,26 @@ void* OwnedImpl::linearize(uint32_t size) {
   }
 }
 
+void OwnedImpl::coalesceOrAddSlice(SlicePtr&& other_slice) {
+  const uint64_t slice_size = other_slice->dataSize();
+  // The `other_slice` content can be coalesced into the existing slice IFF:
+  // 1. The `other_slice` can be coalesced. Objects of type UnownedSlice can not be coalesced. See
+  //    comment in the UnownedSlice class definition;
+  // 2. There are existing slices;
+  // 3. The `other_slice` content length is under the CopyThreshold;
+  // 4. There is enough unused space in the existing slice to accommodate the `other_slice` content.
+  if (other_slice->canCoalesce() && !slices_.empty() && slice_size < CopyThreshold &&
+      slices_.back()->reservableSize() >= slice_size) {
+    // Copy content of the `other_slice`. The `move` methods which call this method effectively
+    // drain the source buffer.
+    addImpl(other_slice->data(), slice_size);
+  } else {
+    // Take ownership of the slice.
+    slices_.emplace_back(std::move(other_slice));
+    length_ += slice_size;
+  }
+}
+
 void OwnedImpl::move(Instance& rhs) {
   ASSERT(&rhs != this);
   ASSERT(isSameBufferImpl(rhs));
@@ -318,10 +357,9 @@ void OwnedImpl::move(Instance& rhs) {
     OwnedImpl& other = static_cast<OwnedImpl&>(rhs);
     while (!other.slices_.empty()) {
       const uint64_t slice_size = other.slices_.front()->dataSize();
-      slices_.emplace_back(std::move(other.slices_.front()));
-      other.slices_.pop_front();
-      length_ += slice_size;
+      coalesceOrAddSlice(std::move(other.slices_.front()));
       other.length_ -= slice_size;
+      other.slices_.pop_front();
     }
     other.postProcess();
   }
@@ -351,9 +389,8 @@ void OwnedImpl::move(Instance& rhs, uint64_t length) {
         other.slices_.front()->drain(copy_size);
         other.length_ -= copy_size;
       } else {
-        slices_.emplace_back(std::move(other.slices_.front()));
+        coalesceOrAddSlice(std::move(other.slices_.front()));
         other.slices_.pop_front();
-        length_ += slice_size;
         other.length_ -= slice_size;
       }
       length -= copy_size;
@@ -651,6 +688,14 @@ bool OwnedImpl::isSameBufferImpl(const Instance& rhs) const {
   return usesOldImpl() == other->usesOldImpl();
 }
 
+std::vector<OwnedSlice::SliceRepresentation> OwnedImpl::describeSlicesForTest() const {
+  std::vector<OwnedSlice::SliceRepresentation> slices;
+  for (const auto& slice : slices_) {
+    slices.push_back(slice->describeSliceForTest());
+  }
+  return slices;
+}
+
 bool OwnedImpl::use_old_impl_ = false;
 
 } // namespace Buffer

source/common/buffer/buffer_impl.h

@@ -174,6 +174,25 @@ class Slice {
     return copy_size;
   }
 
+  /**
+   * Describe the in-memory representation of the slice. For use
+   * in tests that want to make assertions about the specific arrangement of
+   * bytes in a slice.
+   */
+  struct SliceRepresentation {
+    uint64_t data;
+    uint64_t reservable;
+    uint64_t capacity;
+  };
+  SliceRepresentation describeSliceForTest() const {
+    return SliceRepresentation{dataSize(), reservableSize(), capacity_};
+  }
+
+  /**
+   * @return true if content in this Slice can be coalesced into another Slice.
+   */
+  virtual bool canCoalesce() const { return true; }
+
 protected:
   Slice(uint64_t data, uint64_t reservable, uint64_t capacity)
       : data_(data), reservable_(reservable), capacity_(capacity) {}
@@ -401,6 +420,13 @@ class UnownedSlice : public Slice {
 
   ~UnownedSlice() override { fragment_.done(); }
 
+  /**
+   * BufferFragment objects encapsulated by UnownedSlice are used to track when response content
+   * is written into transport connection. As a result these slices can not be coalesced when moved
+   * between buffers.
+   */
+  bool canCoalesce() const override { return false; }
+
 private:
   BufferFragment& fragment_;
 };
@@ -541,6 +567,13 @@ class OwnedImpl : public LibEventInstance {
    */
   static void useOldImpl(bool use_old_impl);
 
+  /**
+   * Describe the in-memory representation of the slices in the buffer. For use
+   * in tests that want to make assertions about the specific arrangement of
+   * bytes in the buffer.
+   */
+  std::vector<OwnedSlice::SliceRepresentation> describeSlicesForTest() const;
+
 private:
   /**
    * @param rhs another buffer
@@ -549,6 +582,15 @@ class OwnedImpl : public LibEventInstance {
    */
   bool isSameBufferImpl(const Instance& rhs) const;
 
+  void addImpl(const void* data, uint64_t size);
+
+  /**
+   * Moves contents of the `other_slice` by either taking its ownership or coalescing it
+   * into an existing slice.
+   * NOTE: the caller is responsible for draining the buffer that contains the `other_slice`.
+   */
+  void coalesceOrAddSlice(SlicePtr&& other_slice);
+
   /** Whether to use the old evbuffer implementation when constructing new OwnedImpl objects. */
   static bool use_old_impl_;