Merge remote-tracking branch 'upstream/main' into template_matching_p…

…ropagate_error

Signed-off-by: Adi Suissa-Peleg <[email protected]>

.bazelrc

@@ -54,9 +54,8 @@ build:docs-ci --action_env=DOCS_RST_CHECK=1 --host_action_env=DOCS_RST_CHECK=1
 build --incompatible_config_setting_private_default_visibility
 build --incompatible_enforce_config_setting_visibility
 
-test --bes_upload_mode=nowait_for_upload_complete
-test --bes_timeout=30s
 test --test_verbose_timeout_warnings
+test --experimental_ui_max_stdouterr_bytes=3048576 #default 1048576
 
 # Allow tags to influence execution requirements
 common --experimental_allow_tags_propagation
@@ -511,6 +510,7 @@ build:rbe-google --config=cache-google
 
 build:rbe-google-bes --bes_backend=grpcs://buildeventservice.googleapis.com
 build:rbe-google-bes --bes_results_url=https://source.cloud.google.com/results/invocations/
+build:rbe-google-bes --bes_upload_mode=fully_async
 
 # RBE (Engflow mobile)
 build:rbe-engflow --google_default_credentials=false

api/envoy/config/core/v3/base.proto

@@ -303,6 +303,59 @@ message RuntimeFeatureFlag {
   string runtime_key = 2 [(validate.rules).string = {min_len: 1}];
 }
 
+message KeyValue {
+  // The key of the key/value pair.
+  string key = 1 [(validate.rules).string = {min_len: 1 max_bytes: 16384}];
+
+  // The value of the key/value pair.
+  bytes value = 2;
+}
+
+// Key/value pair plus option to control append behavior. This is used to specify
+// key/value pairs that should be appended to a set of existing key/value pairs.
+message KeyValueAppend {
+  // Describes the supported actions types for key/value pair append action.
+  enum KeyValueAppendAction {
+    // If the key already exists, this action will result in the following behavior:
+    //
+    // - Comma-concatenated value if multiple values are not allowed.
+    // - New value added to the list of values if multiple values are allowed.
+    //
+    // If the key doesn't exist then this will add pair with specified key and value.
+    APPEND_IF_EXISTS_OR_ADD = 0;
+
+    // This action will add the key/value pair if it doesn't already exist. If the
+    // key already exists then this will be a no-op.
+    ADD_IF_ABSENT = 1;
+
+    // This action will overwrite the specified value by discarding any existing
+    // values if the key already exists. If the key doesn't exist then this will add
+    // the pair with specified key and value.
+    OVERWRITE_IF_EXISTS_OR_ADD = 2;
+
+    // This action will overwrite the specified value by discarding any existing
+    // values if the key already exists. If the key doesn't exist then this will
+    // be no-op.
+    OVERWRITE_IF_EXISTS = 3;
+  }
+
+  // Key/value pair entry that this option to append or overwrite.
+  KeyValue entry = 1 [(validate.rules).message = {required: true}];
+
+  // Describes the action taken to append/overwrite the given value for an existing
+  // key or to only add this key if it's absent.
+  KeyValueAppendAction action = 2 [(validate.rules).enum = {defined_only: true}];
+}
+
+// Key/value pair to append or remove.
+message KeyValueMutation {
+  // Key/value pair to append or overwrite. Only one of ``append`` or ``remove`` can be set.
+  KeyValueAppend append = 1;
+
+  // Key to remove. Only one of ``append`` or ``remove`` can be set.
+  string remove = 2 [(validate.rules).string = {max_bytes: 16384}];
+}
+
 // Query parameter name/value pair.
 message QueryParameter {
   // The key of the query parameter. Case sensitive.

bazel/test_for_benchmark_wrapper.sh

@@ -3,4 +3,4 @@
 # Set the benchmark time to 0 to just verify that the benchmark runs to
 # completion.  We're interacting with two different flag parsers, so the order
 # of flags and the -- matters.
-"${TEST_SRCDIR}/envoy/${1}" "${@:2}" --skip_expensive_benchmarks -- --benchmark_min_time=0
+"${TEST_SRCDIR}/envoy/${1}" "${@:2}" --skip_expensive_benchmarks -- --benchmark_min_time=0s

changelogs/current.yaml

@@ -41,8 +41,9 @@ minor_behavior_changes:
 # *Changes that may cause incompatibilities for some users, but should not for most*
 - area: grpc
   change: |
-    Changes in ``AsyncStreamImpl`` now propagate tracing context headers in bidirectional streams when using
-    :ref:`Envoy gRPC client <envoy_v3_api_field_config.core.v3.GrpcService.envoy_grpc>`. Previously, tracing context headers
+    Changes in ``AsyncStreamImpl`` and ``GoogleAsyncStreamImpl`` now propagate tracing context headers in bidirectional streams when using
+    :ref:`Envoy gRPC client <envoy_v3_api_field_config.core.v3.GrpcService.envoy_grpc>` or
+    :ref:`Google C++ gRPC client <envoy_v3_api_field_config.core.v3.GrpcService.google_grpc>`. Previously, tracing context headers
     were not being set when calling external services such as ``ext_proc``.
 - area: http
   change: |
@@ -174,6 +175,11 @@ bug_fixes:
   change: |
     Added one option to disable the response body buffering for mirror request. Also introduced a 32MB cap for the response
     buffer, which can be changed by the runtime flag ``http.async_response_buffer_limit`` based on the product needs.
+- area: ext_authz
+  change: |
+    Validate http service path_prefix
+    :ref:`path_prefix <envoy_v3_api_field_extensions.filters.http.ext_authz.v3.HttpService.path_prefix>`,
+    Validate http service path_prefix configuration must start with ``/``.
 
 removed_config_or_runtime:
 # *Normally occurs at the end of the* :ref:`deprecation period <deprecated>`
@@ -183,6 +189,12 @@ removed_config_or_runtime:
 - area: http
   change: |
     Removed ``envoy.reloadable_features.use_cluster_cache_for_alt_protocols_filter`` runtime flag and lagacy code paths.
+- area: load_balancing
+  change: |
+    Removed ``envoy.reloadable_features.enable_zone_routing_different_zone_counts`` runtime flag and legacy code paths.
+- area: load_balancing
+  change: |
+    Removed ``envoy.reloadable_features.locality_routing_use_new_routing_logic`` runtime flag and legacy code paths.
 - area: http
   change: |
     Removed ``envoy.reloadable_features.proxy_status_upstream_request_timeout`` runtime flag and lagacy code paths.
@@ -198,9 +210,15 @@ removed_config_or_runtime:
 - area: http
   change: |
     Removed ``envoy.reloadable_features.http_allow_partial_urls_in_referer`` runtime flag and legacy code paths.
+- area: oauth
+  change: |
+    Removed ``envoy.reloadable_features.oauth_make_token_cookie_httponly`` runtime flag and legacy code paths.
 - area: http
   change: |
     Removed ``envoy.reloadable_features.lowercase_scheme`` runtime flag and legacy code paths.
+- area: oauth
+  change: |
+    Removed ``envoy.reloadable_features.hmac_base64_encoding_only`` runtime flag and legacy code paths.
 - area: upstream
   change: |
     Removed ``envoy.reloadable_features.convert_legacy_lb_config`` runtime flag and legacy code paths.

contrib/generic_proxy/filters/network/source/codecs/dubbo/config.h

@@ -28,6 +28,16 @@ class DubboRequest : public Request {
     ASSERT(inner_metadata_ != nullptr);
     ASSERT(inner_metadata_->hasContext());
     ASSERT(inner_metadata_->hasRequest());
+
+    uint32_t frame_flags = FrameFlags::FLAG_END_STREAM; // Dubbo message only has one frame.
+    if (!inner_metadata_->context().isTwoWay()) {
+      frame_flags |= FrameFlags::FLAG_ONE_WAY;
+    }
+    if (inner_metadata_->context().heartbeat()) {
+      frame_flags |= FrameFlags::FLAG_HEARTBEAT;
+    }
+
+    stream_frame_flags_ = {static_cast<uint64_t>(inner_metadata_->requestId()), frame_flags};
   }
 
   // Request
@@ -43,9 +53,10 @@ class DubboRequest : public Request {
   // StreamFrame
   FrameFlags frameFlags() const override { return stream_frame_flags_; }
 
-  FrameFlags stream_frame_flags_;
-
   Common::Dubbo::MessageMetadataSharedPtr inner_metadata_;
+
+private:
+  FrameFlags stream_frame_flags_;
 };
 
 class DubboResponse : public Response {
@@ -56,6 +67,16 @@ class DubboResponse : public Response {
     ASSERT(inner_metadata_->hasContext());
     ASSERT(inner_metadata_->hasResponse());
     refreshStatus();
+
+    uint32_t frame_flags = FrameFlags::FLAG_END_STREAM; // Dubbo message only has one frame.
+    if (!inner_metadata_->context().isTwoWay()) {
+      frame_flags |= FrameFlags::FLAG_ONE_WAY;
+    }
+    if (inner_metadata_->context().heartbeat()) {
+      frame_flags |= FrameFlags::FLAG_HEARTBEAT;
+    }
+
+    stream_frame_flags_ = {static_cast<uint64_t>(inner_metadata_->requestId()), frame_flags};
   }
 
   void refreshStatus();
@@ -67,10 +88,11 @@ class DubboResponse : public Response {
   // StreamFrame
   FrameFlags frameFlags() const override { return stream_frame_flags_; }
 
-  FrameFlags stream_frame_flags_;
-
   StreamStatus status_;
   Common::Dubbo::MessageMetadataSharedPtr inner_metadata_;
+
+private:
+  FrameFlags stream_frame_flags_;
 };
 
 class DubboCodecBase : public Logger::Loggable<Logger::Id::connection> {
@@ -136,12 +158,9 @@ class DubboDecoderBase : public DubboCodecBase, public CodecType {
         return Common::Dubbo::DecodeStatus::Success;
       }
 
-      auto message = std::make_unique<DecoderMessageType>(metadata_);
-      message->stream_frame_flags_ = {{static_cast<uint64_t>(metadata_->requestId()),
-                                       !metadata_->context().isTwoWay(), false,
-                                       metadata_->context().heartbeat()},
-                                      true};
+      auto message = std::make_unique<DecoderMessageType>(std::move(metadata_));
       metadata_.reset();
+
       callback_->onDecodingSuccess(std::move(message));
 
       return Common::Dubbo::DecodeStatus::Success;

contrib/generic_proxy/filters/network/source/codecs/http1/config.h

@@ -60,7 +60,7 @@ class HttpRequestFrame : public HttpHeaderFrame<StreamRequest> {
   HttpRequestFrame(Http::RequestHeaderMapPtr request, bool end_stream)
       : request_(std::move(request)) {
     ASSERT(request_ != nullptr);
-    frame_flags_ = {StreamFlags{}, end_stream};
+    frame_flags_ = {0, end_stream ? FrameFlags::FLAG_END_STREAM : FrameFlags::FLAG_EMPTY};
   }
 
   absl::string_view host() const override { return request_->getHostValue(); }
@@ -80,7 +80,15 @@ class HttpResponseFrame : public HttpHeaderFrame<StreamResponse> {
     const bool drain_close = Envoy::StringUtil::caseFindToken(
         response_->getConnectionValue(), ",", Http::Headers::get().ConnectionValues.Close);
 
-    frame_flags_ = {StreamFlags{0, false, drain_close, false}, end_stream};
+    uint32_t flags = 0;
+    if (end_stream) {
+      flags |= FrameFlags::FLAG_END_STREAM;
+    }
+    if (drain_close) {
+      flags |= FrameFlags::FLAG_DRAIN_CLOSE;
+    }
+
+    frame_flags_ = {0, flags};
   }
 
   StreamStatus status() const override {
@@ -101,7 +109,7 @@ class HttpResponseFrame : public HttpHeaderFrame<StreamResponse> {
 class HttpRawBodyFrame : public CommonFrame {
 public:
   HttpRawBodyFrame(Envoy::Buffer::Instance& buffer, bool end_stream)
-      : frame_flags_({StreamFlags{}, end_stream}) {
+      : frame_flags_(0, end_stream ? FrameFlags::FLAG_END_STREAM : FrameFlags::FLAG_EMPTY) {
     buffer_.move(buffer);
   }
   FrameFlags frameFlags() const override { return frame_flags_; }

contrib/generic_proxy/filters/network/source/codecs/kafka/config.h

@@ -28,8 +28,7 @@ class KafkaRequestFrame : public GenericProxy::StreamRequest {
     if (request_ == nullptr) {
       return FrameFlags{};
     }
-    return FrameFlags{
-        StreamFlags{static_cast<uint64_t>(request_->request_header_.correlation_id_)}};
+    return FrameFlags{static_cast<uint64_t>(request_->request_header_.correlation_id_)};
   }
 
   absl::string_view protocol() const override { return "kafka"; }
@@ -46,7 +45,7 @@ class KafkaResponseFrame : public GenericProxy::StreamResponse {
     if (response_ == nullptr) {
       return FrameFlags{};
     }
-    return FrameFlags{StreamFlags{static_cast<uint64_t>(response_->metadata_.correlation_id_)}};
+    return FrameFlags{static_cast<uint64_t>(response_->metadata_.correlation_id_)};
   }
 
   absl::string_view protocol() const override { return "kafka"; }

contrib/generic_proxy/filters/network/source/config.cc

@@ -12,6 +12,38 @@ namespace Extensions {
 namespace NetworkFilters {
 namespace GenericProxy {
 
+template <class Context>
+static AccessLog::FilterBasePtr<Context>
+accessLogFilterFromProto(const envoy::config::accesslog::v3::AccessLogFilter& config,
+                         Server::Configuration::FactoryContext& context) {
+  if (!config.has_extension_filter()) {
+    ExceptionUtil::throwEnvoyException(
+        "Access log filter: only extension filter is supported by non-HTTP access loggers.");
+  }
+
+  auto& factory =
+      Config::Utility::getAndCheckFactory<Envoy::AccessLog::ExtensionFilterFactoryBase<Context>>(
+          config.extension_filter());
+  return factory.createFilter(config.extension_filter(), context);
+}
+
+template <class Context>
+static AccessLog::InstanceBaseSharedPtr<Context>
+accessLoggerFromProto(const envoy::config::accesslog::v3::AccessLog& config,
+                      Server::Configuration::FactoryContext& context) {
+  AccessLog::FilterBasePtr<Context> filter;
+  if (config.has_filter()) {
+    filter = accessLogFilterFromProto<Context>(config.filter(), context);
+  }
+
+  auto& factory =
+      Config::Utility::getAndCheckFactory<Envoy::AccessLog::AccessLogInstanceFactoryBase<Context>>(
+          config);
+  ProtobufTypes::MessagePtr message = Config::Utility::translateToFactoryConfig(
+      config, context.messageValidationVisitor(), factory);
+
+  return factory.createAccessLogInstance(*message, std::move(filter), context);
+}
 SINGLETON_MANAGER_REGISTRATION(generic_route_config_provider_manager);
 SINGLETON_MANAGER_REGISTRATION(generic_proxy_code_or_flag_stats);
 
@@ -127,7 +159,7 @@ Factory::createFilterFactoryFromProtoTyped(const ProxyConfig& proto_config,
   std::vector<AccessLogInstanceSharedPtr> access_logs;
   for (const auto& access_log : proto_config.access_log()) {
     AccessLogInstanceSharedPtr current_access_log =
-        AccessLog::AccessLogFactory::accessLoggerFromProto<FormatterContext>(access_log, context);
+        accessLoggerFromProto<FormatterContext>(access_log, context);
     access_logs.push_back(current_access_log);
   }