From a9aed218c3f650f085fc3ff789e0a59c7f9b8115 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 25 Oct 2022 16:45:44 +0100 Subject: [PATCH 001/112] repo: Update `protoc-gen-validate` links -> bufbuild (#23495) Signed-off-by: Ryan Northey --- DEPENDENCY_POLICY.md | 2 +- api/API_VERSIONING.md | 2 +- api/STYLE.md | 2 +- api/envoy/api/v2/core/grpc_service.proto | 2 +- api/envoy/config/core/v3/grpc_service.proto | 2 +- bazel/dependency_imports.bzl | 10 +++++----- docs/root/api-docs/xds_protocol.rst | 2 +- source/common/common/regex.cc | 2 +- source/common/protobuf/utility.h | 2 +- source/common/upstream/upstream_impl.cc | 2 +- .../local_ratelimit/local_ratelimit_fuzz_test.cc | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/DEPENDENCY_POLICY.md b/DEPENDENCY_POLICY.md index 892b99ae7e63..17c1d641bfd7 100644 --- a/DEPENDENCY_POLICY.md +++ b/DEPENDENCY_POLICY.md @@ -156,4 +156,4 @@ The following dependencies are exempt from the policy: * Any developer-only facing tooling or the documentation build. * Transitive build time dependencies, e.g. Go projects vendored into - [protoc-gen-validate](https://github.com/envoyproxy/protoc-gen-validate). + [protoc-gen-validate](https://github.com/bufbuild/protoc-gen-validate). diff --git a/api/API_VERSIONING.md b/api/API_VERSIONING.md index 88be50a6a1c2..f99f2aacc103 100644 --- a/api/API_VERSIONING.md +++ b/api/API_VERSIONING.md @@ -63,7 +63,7 @@ experience a backward compatible break on a change. Specifically: churn. * Increasing the strictness of - [protoc-gen-validate](https://github.com/envoyproxy/protoc-gen-validate) annotations. Exceptions + [protoc-gen-validate](https://github.com/bufbuild/protoc-gen-validate) annotations. Exceptions may be granted for scenarios in which these stricter conditions model behavior already implied structurally or by documentation. diff --git a/api/STYLE.md b/api/STYLE.md index 04e7b4ee2fb2..b0821c927be2 100644 --- a/api/STYLE.md +++ b/api/STYLE.md @@ -184,7 +184,7 @@ metadata. We describe these annotations below by category. the field will be promoted to a given `oneof` in the next API major version. * `[(udpa.annotations.sensitive) = true]` to denote sensitive fields that should be redacted in output such as logging or configuration dumps. -* [PGV annotations](https://github.com/envoyproxy/protoc-gen-validate) to denote field +* [PGV annotations](https://github.com/bufbuild/protoc-gen-validate) to denote field value constraints. ### Enum value level diff --git a/api/envoy/api/v2/core/grpc_service.proto b/api/envoy/api/v2/core/grpc_service.proto index faafb7f0f7f0..027a45e4c5ee 100644 --- a/api/envoy/api/v2/core/grpc_service.proto +++ b/api/envoy/api/v2/core/grpc_service.proto @@ -100,7 +100,7 @@ message GrpcService { message StsService { // URI of the token exchange service that handles token exchange requests. // [#comment:TODO(asraa): Add URI validation when implemented. Tracked by - // https://github.com/envoyproxy/protoc-gen-validate/issues/303] + // https://github.com/bufbuild/protoc-gen-validate/issues/303] string token_exchange_service_uri = 1; // Location of the target service or resource where the client diff --git a/api/envoy/config/core/v3/grpc_service.proto b/api/envoy/config/core/v3/grpc_service.proto index fe8be7672ab0..6027b2b47d12 100644 --- a/api/envoy/config/core/v3/grpc_service.proto +++ b/api/envoy/config/core/v3/grpc_service.proto @@ -142,7 +142,7 @@ message GrpcService { // URI of the token exchange service that handles token exchange requests. // [#comment:TODO(asraa): Add URI validation when implemented. Tracked by - // https://github.com/envoyproxy/protoc-gen-validate/issues/303] + // https://github.com/bufbuild/protoc-gen-validate/issues/303] string token_exchange_service_uri = 1; // Location of the target service or resource where the client diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index 29a63157e17f..b62d3cf28443 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -79,7 +79,7 @@ def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, y # project_url = "https://pkg.go.dev/golang.org/x/net", # last_update = "2020-02-26" # use_category = ["api"], - # source = "https://github.com/envoyproxy/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L129-L134" + # source = "https://github.com/bufbuild/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L129-L134" ) go_repository( name = "org_golang_x_text", @@ -90,7 +90,7 @@ def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, y # project_url = "https://pkg.go.dev/golang.org/x/text", # last_update = "2021-06-16" # use_category = ["api"], - # source = "https://github.com/envoyproxy/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L148-L153" + # source = "https://github.com/bufbuild/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L148-L153" ) go_repository( name = "com_github_spf13_afero", @@ -101,7 +101,7 @@ def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, y # project_url = "https://pkg.go.dev/github.com/spf13/afero", # last_update = "2021-03-20" # use_category = ["api"], - # source = "https://github.com/envoyproxy/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L60-L65" + # source = "https://github.com/bufbuild/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L60-L65" ) go_repository( name = "com_github_lyft_protoc_gen_star", @@ -112,7 +112,7 @@ def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, y # project_url = "https://pkg.go.dev/github.com/lyft/protoc-gen-star", # last_update = "2022-03-04" # use_category = ["api"], - # source = "https://github.com/envoyproxy/protoc-gen-validate/blob/v0.6.7/dependencies.bzl#L35-L40" + # source = "https://github.com/bufbuild/protoc-gen-validate/blob/v0.6.7/dependencies.bzl#L35-L40" ) go_repository( name = "com_github_iancoleman_strcase", @@ -123,5 +123,5 @@ def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, y # project_url = "https://pkg.go.dev/github.com/iancoleman/strcase", # last_update = "2020-11-22" # use_category = ["api"], - # source = "https://github.com/envoyproxy/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L23-L28" + # source = "https://github.com/bufbuild/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L23-L28" ) diff --git a/docs/root/api-docs/xds_protocol.rst b/docs/root/api-docs/xds_protocol.rst index c6070a8e39b2..8a1339d42abf 100644 --- a/docs/root/api-docs/xds_protocol.rst +++ b/docs/root/api-docs/xds_protocol.rst @@ -41,7 +41,7 @@ Protoc-Gen-Validate Annotations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The protobuf messages for the individual xDS resource types have annotations -using `protoc-gen-validate `_ +using `protoc-gen-validate `_ (PGV), which indicate semantic constraints to be used to validate the contents of a resource when it is received by a client. diff --git a/source/common/common/regex.cc b/source/common/common/regex.cc index d0a91fd54e66..679efd3029a3 100644 --- a/source/common/common/regex.cc +++ b/source/common/common/regex.cc @@ -75,7 +75,7 @@ ProtobufTypes::MessagePtr GoogleReEngineFactory::createEmptyConfigProto() { REGISTER_FACTORY(GoogleReEngineFactory, EngineFactory); std::regex Utility::parseStdRegex(const std::string& regex, std::regex::flag_type flags) { - // TODO(zuercher): In the future, PGV (https://github.com/envoyproxy/protoc-gen-validate) + // TODO(zuercher): In the future, PGV (https://github.com/bufbuild/protoc-gen-validate) // annotations may allow us to remove this in favor of direct validation of regular // expressions. try { diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index 605f86e8ed03..36c4d45a67ab 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -117,7 +117,7 @@ uint64_t fractionalPercentDenominatorToInt( // @param default_value supplies the default if the field is not present. // // TODO(anirudhmurali): Recommended to capture and validate NaN values in PGV -// Issue: https://github.com/envoyproxy/protoc-gen-validate/issues/85 +// Issue: https://github.com/bufbuild/protoc-gen-validate/issues/85 #define PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(message, field_name, max_value, \ default_value) \ ([](const auto& msg) { \ diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 8e7014a23767..600161b4b904 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1106,7 +1106,7 @@ ClusterInfoImpl::ClusterInfoImpl( } // TODO(htuch): Remove this temporary workaround when we have - // https://github.com/envoyproxy/protoc-gen-validate/issues/97 resolved. This just provides + // https://github.com/bufbuild/protoc-gen-validate/issues/97 resolved. This just provides // early validation of sanity of fields that we should catch at config ingestion. DurationUtil::durationToMilliseconds(common_lb_config_.update_merge_window()); diff --git a/test/extensions/filters/network/local_ratelimit/local_ratelimit_fuzz_test.cc b/test/extensions/filters/network/local_ratelimit/local_ratelimit_fuzz_test.cc index 34d550782065..59549020ec91 100644 --- a/test/extensions/filters/network/local_ratelimit/local_ratelimit_fuzz_test.cc +++ b/test/extensions/filters/network/local_ratelimit/local_ratelimit_fuzz_test.cc @@ -51,7 +51,7 @@ DEFINE_PROTO_FUZZER( // TODO: // protoc-gen-validate has an issue on type "Duration" which may generate interval with seconds // > 0 while "nanos" < 0. And negative "nanos" will cause validation inside the filter to fail. - // see https://github.com/envoyproxy/protoc-gen-validate/issues/348 for detail. + // see https://github.com/bufbuild/protoc-gen-validate/issues/348 for detail. ENVOY_LOG_MISC(debug, "In fill_interval, seconds or nanos should not be negative! Exception: {}", e.what()); From 367a81d753bc054439db434441dc9b9aa6deca2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Oct 2022 21:18:42 -0400 Subject: [PATCH 002/112] build(deps): bump colorama from 0.4.5 to 0.4.6 in /tools/dependency (#23655) Bumps [colorama](https://github.com/tartley/colorama) from 0.4.5 to 0.4.6. - [Release notes](https://github.com/tartley/colorama/releases) - [Changelog](https://github.com/tartley/colorama/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tartley/colorama/compare/0.4.5...0.4.6) --- updated-dependencies: - dependency-name: colorama dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/dependency/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/dependency/requirements.txt b/tools/dependency/requirements.txt index 326f41d2318d..6eab8f9ab073 100644 --- a/tools/dependency/requirements.txt +++ b/tools/dependency/requirements.txt @@ -164,9 +164,9 @@ charset-normalizer==3.0.0 \ --hash=sha256:f810dc652bda19caf9ee2427325c73f5983ca4e014aaabcfb9983cb452863812 \ --hash=sha256:ff5cf1e500a0da22f385691a1680a78d5e53506aadc374d722c645de6449f7f0 # via requests -colorama==0.4.5 \ - --hash=sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da \ - --hash=sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4 +colorama==0.4.6 \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 # via -r requirements.in deprecated==1.2.13 \ --hash=sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d \ From 020201ff14c38ac541abf575ee8e366aa6e1d32c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Oct 2022 02:33:10 +0000 Subject: [PATCH 003/112] build(deps): bump orjson from 3.8.0 to 3.8.1 in /tools/base (#23657) Bumps [orjson](https://github.com/ijl/orjson) from 3.8.0 to 3.8.1. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.8.0...3.8.1) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 93 ++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index d25849d12cef..151a0b0a0610 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -649,49 +649,56 @@ multidict==6.0.2 \ # -r requirements.in # aiohttp # yarl -orjson==3.8.0 \ - --hash=sha256:02d638d43951ba346a80f0abd5942a872cc87db443e073f6f6fc530fee81e19b \ - --hash=sha256:03ed95814140ff09f550b3a42e6821f855d981c94d25b9cc83e8cca431525d70 \ - --hash=sha256:1b1cd25acfa77935bb2e791b75211cec0cfc21227fe29387e553c545c3ff87e1 \ - --hash=sha256:2058653cc12b90e482beacb5c2d52dc3d7606f9e9f5a52c1c10ef49371e76f52 \ - --hash=sha256:2065b6d280dc58f131ffd93393737961ff68ae7eb6884b68879394074cc03c13 \ - --hash=sha256:25b5e48fbb9f0b428a5e44cf740675c9281dd67816149fc33659803399adbbe8 \ - --hash=sha256:2bdb1042970ca5f544a047d6c235a7eb4acdb69df75441dd1dfcbc406377ab37 \ - --hash=sha256:2d81e6e56bbea44be0222fb53f7b255b4e7426290516771592738ca01dbd053b \ - --hash=sha256:3c7225e8b08996d1a0c804d3a641a53e796685e8c9a9fd52bd428980032cad9a \ - --hash=sha256:3e2459d441ab8fd8b161aa305a73d5269b3cda13b5a2a39eba58b4dd3e394f49 \ - --hash=sha256:4065906ce3ad6195ac4d1bddde862fe811a42d7be237a1ff762666c3a4bb2151 \ - --hash=sha256:5b072ef8520cfe7bd4db4e3c9972d94336763c2253f7c4718a49e8733bada7b8 \ - --hash=sha256:5edb93cdd3eb32977633fa7aaa6a34b8ab54d9c49cdcc6b0d42c247a29091b22 \ - --hash=sha256:5f856279872a4449fc629924e6a083b9821e366cf98b14c63c308269336f7c14 \ - --hash=sha256:5fd6cac83136e06e538a4d17117eaeabec848c1e86f5742d4811656ad7ee475f \ - --hash=sha256:6433c956f4a18112342a18281e0bec67fcd8b90be3a5271556c09226e045d805 \ - --hash=sha256:655d7387a1634a9a477c545eea92a1ee902ab28626d701c6de4914e2ed0fecd2 \ - --hash=sha256:66c19399bb3b058e3236af7910b57b19a4fc221459d722ed72a7dc90370ca090 \ - --hash=sha256:6a23b40c98889e9abac084ce5a1fb251664b41da9f6bdb40a4729e2288ed2ed4 \ - --hash=sha256:6e3da2e4bd27c3b796519ca74132c7b9e5348fb6746315e0f6c1592bc5cf1caf \ - --hash=sha256:6ea5fe20ef97545e14dd4d0263e4c5c3bc3d2248d39b4b0aed4b84d528dfc0af \ - --hash=sha256:7536a2a0b41672f824912aeab545c2467a9ff5ca73a066ff04fb81043a0a177a \ - --hash=sha256:7990a9caf3b34016ac30be5e6cfc4e7efd76aa85614a1215b0eae4f0c7e3db59 \ - --hash=sha256:7b0e72974a5d3b101226899f111368ec2c9824d3e9804af0e5b31567f53ad98a \ - --hash=sha256:87462791dd57de2e3e53068bf4b7169c125c50960f1bdda08ed30c797cb42a56 \ - --hash=sha256:896a21a07f1998648d9998e881ab2b6b80d5daac4c31188535e9d50460edfcf7 \ - --hash=sha256:8b391d5c2ddc2f302d22909676b306cb6521022c3ee306c861a6935670291b2c \ - --hash=sha256:8f687776a03c19f40b982fb5c414221b7f3d19097841571be2223d1569a59877 \ - --hash=sha256:9a93850a1bdc300177b111b4b35b35299f046148ba23020f91d6efd7bf6b9d20 \ - --hash=sha256:9e6ac22cec72d5b39035b566e4b86c74b84866f12b5b0b6541506a080fb67d6d \ - --hash=sha256:a709c2249c1f2955dbf879506fd43fa08c31fdb79add9aeb891e3338b648bf60 \ - --hash=sha256:b68a42a31f8429728183c21fb440c21de1b62e5378d0d73f280e2d894ef8942e \ - --hash=sha256:be02f6acee33bb63862eeff80548cd6b8a62e2d60ad2d8dfd5a8824cc43d8887 \ - --hash=sha256:d189e2acb510e374700cb98cf11b54f0179916ee40f8453b836157ae293efa79 \ - --hash=sha256:d2b5dafbe68237a792143137cba413447f60dd5df428e05d73dcba10c1ea6fcf \ - --hash=sha256:e1418feeb8b698b9224b1f024555895169d481604d5d884498c1838d7412794c \ - --hash=sha256:e2defd9527651ad39ec20ae03c812adf47ef7662bdd6bc07dabb10888d70dc62 \ - --hash=sha256:e2f4a5542f50e3d336a18cb224fc757245ca66b1fd0b70b5dd4471b8ff5f2b0e \ - --hash=sha256:e68c699471ea3e2dd1b35bfd71c6a0a0e4885b64abbe2d98fce1ef11e0afaff3 \ - --hash=sha256:f4b46dbdda2f0bd6480c39db90b21340a19c3b0fcf34bc4c6e465332930ca539 \ - --hash=sha256:fb42f7cf57d5804a9daa6b624e3490ec9e2631e042415f3aebe9f35a8492ba6c \ - --hash=sha256:ff13410ddbdda5d4197a4a4c09969cb78c722a67550f0a63c02c07aadc624833 +orjson==3.8.1 \ + --hash=sha256:03389e3750c521a7f3d4837de23cfd21a7f24574b4b3985c9498f440d21adb03 \ + --hash=sha256:07c42de52dfef56cdcaf2278f58e837b26f5b5af5f1fd133a68c4af203851fc7 \ + --hash=sha256:0b4e3857dd2416b479f700e9bdf4fcec8c690d2716622397d2b7e848f9833e50 \ + --hash=sha256:0bd5b4e539db8a9635776bdf9a25c3db84e37165e65d45c8ca90437adc46d6d8 \ + --hash=sha256:0f21eed14697083c01f7e00a87e21056fc8fb5851e8a7bca98345189abcdb4d4 \ + --hash=sha256:124207d2cd04e845eaf2a6171933cde40aebcb8c2d7d3b081e01be066d3014b6 \ + --hash=sha256:21efb87b168066201a120b0f54a2381f6f51ff3727e07b3908993732412b314a \ + --hash=sha256:231c30958ed99c23128a21993c5ac0a70e1e568e6a898a47f70d5d37461ca47c \ + --hash=sha256:395d02fd6be45f960da014372e7ecefc9e5f8df57a0558b7111a5fa8423c0669 \ + --hash=sha256:3fd5472020042482d7da4c26a0ee65dbd931f691e1c838c6cf4232823179ecc1 \ + --hash=sha256:4449e70b98f3ad3e43958360e4be1189c549865c0a128e8629ec96ce92d251c3 \ + --hash=sha256:45357eea9114bd41ef19280066591e9069bb4f6f5bffd533e9bfc12a439d735f \ + --hash=sha256:45c1914795ffedb2970bfcd3ed83daf49124c7c37943ed0a7368971c6ea5e278 \ + --hash=sha256:4f5a9bc5bc4d730153529cb0584c63ff286d50663ccd48c9435423660b1bb12d \ + --hash=sha256:59b4baf71c9f39125d7e535974b146cc180926462969f6d8821b4c5e975e11b3 \ + --hash=sha256:5a9e324213220578d324e0858baeab47808a13d3c3fbc6ba55a3f4f069d757cf \ + --hash=sha256:5ded261268d5dfd307078fe3370295e5eb15bdde838bbb882acf8538e061c451 \ + --hash=sha256:5e3db6496463c3000d15b7a712da5a9601c6c43682f23f81862fe1d2a338f295 \ + --hash=sha256:6071bcf51f0ae4d53b9d3e9164f7138164df4291c484a7b14562075aaa7a2b7b \ + --hash=sha256:6802edf98f6918e89df355f56be6e7db369b31eed64ff2496324febb8b0aa43b \ + --hash=sha256:69097c50c3ccbcc61292192b045927f1688ca57ce80525dc5d120e0b91e19bb0 \ + --hash=sha256:6956cf7a1ac97523e96f75b11534ff851df99a6474a561ad836b6e82004acbb8 \ + --hash=sha256:6a7b76d4b44bca418f7797b1e157907b56b7d31caa9091db4e99ebee51c16933 \ + --hash=sha256:7adaac93678ac61f5dc070f615b18639d16ee66f6a946d5221dbf315e8b74bec \ + --hash=sha256:8623ac25fa0850a44ac845e9333c4da9ae5707b7cec8ac87cbe9d4e41137180f \ + --hash=sha256:8f672f3987f6424f60ab2e86ea7ed76dd2806b8e9b506a373fc8499aed85ddb5 \ + --hash=sha256:97839a6abbebb06099294e6057d5b3061721ada08b76ae792e7041b6cb54c97f \ + --hash=sha256:a4244f4199a160717f0027e434abb886e322093ceadb2f790ff0c73ed3e17662 \ + --hash=sha256:a70aaa2e56356e58c6e1b49f7b7f069df5b15e55db002a74db3ff3f7af67c7ff \ + --hash=sha256:a806aca6b80fa1d996aa16593e4995a71126a085ee1a59fff19ccad29a4e47fd \ + --hash=sha256:b0c1750f73658906b82cabbf4be2f74300644c17cb037fbc8b48d746c3b90c76 \ + --hash=sha256:b0f9d9b5c6692097de07dd0b2d5ff20fd135bacd1b2fb7ea383ee717a4150c93 \ + --hash=sha256:b9abc49c014def1b832fcd53bdc670474b6fe41f373d16f40409882c0d0eccba \ + --hash=sha256:c15e7d691cee75b5192fc1fa8487bf541d463246dc25c926b9b40f5b6ab56770 \ + --hash=sha256:c2c9ef10b6344465fd5ac002be2d34f818211274dd79b44c75b2c14a979f84f3 \ + --hash=sha256:caff3c1e964cfee044a03a46244ecf6373f3c56142ad16458a1446ac6d69824a \ + --hash=sha256:d45db052d01d0ab7579470141d5c3592f4402d43cfacb67f023bc1210a67b7bc \ + --hash=sha256:d67a0bd0283a3b17ac43c5ab8e4a7e9d3aa758d6ec5d51c232343c408825a5ad \ + --hash=sha256:d89ef8a4444d83e0a5171d14f2ab4895936ab1773165b020f97d29cf289a2d88 \ + --hash=sha256:d8ed77098c2e22181fce971f49a34204c38b79ca91c01d515d07015339ae8165 \ + --hash=sha256:da6306e1f03e7085fe0db61d4a3377f70c6fd865118d0afe17f80ae9a8f6f124 \ + --hash=sha256:e073338e422f518c1d4d80efc713cd17f3ed6d37c8c7459af04a95459f3206d1 \ + --hash=sha256:e2aae92398c0023ac26a6cd026375f765ef5afe127eccabf563c78af7b572d59 \ + --hash=sha256:e399ed1b0d6f8089b9b6ff2cb3e71ba63a56d8ea88e1d95467949795cc74adfd \ + --hash=sha256:e7822cba140f7ca48ed0256229f422dbae69e3a3475176185db0c0538cfadb57 \ + --hash=sha256:f532c2cbe8c140faffaebcfb34d43c9946599ea8138971f181a399bec7d6b123 \ + --hash=sha256:f850489d89ea12be486492e68f0fd63e402fa28e426d4f0b5fc1eec0595e6109 \ + --hash=sha256:f8873e490dea0f9cd975d66f84618b6fb57b1ba45ecb218313707a71173d764f \ + --hash=sha256:fe25f50dc3d45364428baa0dbe3f613a5171c64eb0286eb775136b74e61ba58a # via # -r requirements.in # aio-core From badf51d1449f2d7057cbaa354bb2cc7ffbb68567 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Oct 2022 08:48:25 +0100 Subject: [PATCH 004/112] build(deps): bump mysql from `12bae50` to `06314a7` in /examples/mysql (#23664) Bumps mysql from `12bae50` to `06314a7`. --- updated-dependencies: - dependency-name: mysql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/mysql/Dockerfile-mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mysql/Dockerfile-mysql b/examples/mysql/Dockerfile-mysql index 18ad035cc9b2..d59d5c1bbcb8 100644 --- a/examples/mysql/Dockerfile-mysql +++ b/examples/mysql/Dockerfile-mysql @@ -1 +1 @@ -FROM mysql:8.0.31@sha256:12bae50f531fef9dc7726072446cd7c4b461eaa154611659c891a0d9f628684f +FROM mysql:8.0.31@sha256:06314a7a220f6043436cfd72fd9c7f174fd58ef69fe4b788625fa53be4ab66aa From 0876a0792467ce7c4d7ca7b8c937834e3a215700 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Oct 2022 08:48:33 +0100 Subject: [PATCH 005/112] build(deps): bump postgres from `7694225` to `9653626` in /examples/shared/postgres (#23665) build(deps): bump postgres in /examples/shared/postgres Bumps postgres from `7694225` to `9653626`. --- updated-dependencies: - dependency-name: postgres dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/postgres/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/postgres/Dockerfile b/examples/shared/postgres/Dockerfile index 2bcb7b14d9e7..d4e28cdec103 100644 --- a/examples/shared/postgres/Dockerfile +++ b/examples/shared/postgres/Dockerfile @@ -1 +1 @@ -FROM postgres:latest@sha256:769422529210357f359e7e5620246028837f84fabe92c838ca2aef75fe2e0741 +FROM postgres:latest@sha256:9653626520498d747b798ed241d2d52248e2918684164b48ee49302326891a84 From deea56ffe4cb92393a7aa60894a20a1476ca7f96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Oct 2022 08:51:16 +0100 Subject: [PATCH 006/112] build(deps): bump node from `56b132f` to `5a7b677` in /examples/ext_authz/auth/http-service (#23660) build(deps): bump node in /examples/ext_authz/auth/http-service Bumps node from `56b132f` to `5a7b677`. --- updated-dependencies: - dependency-name: node dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/ext_authz/auth/http-service/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ext_authz/auth/http-service/Dockerfile b/examples/ext_authz/auth/http-service/Dockerfile index 91c1212da3d4..6a014e395279 100644 --- a/examples/ext_authz/auth/http-service/Dockerfile +++ b/examples/ext_authz/auth/http-service/Dockerfile @@ -1,4 +1,4 @@ -FROM node:alpine@sha256:56b132fd12038c90041deb314b2110fcfbf62b52db5ba861a88c7af1077a5ece +FROM node:alpine@sha256:5a7b6772549bfbb856769f9e3d090812450399a746654a8b89a80b2026591902 COPY . /app CMD ["node", "/app/http-service/server"] From 735759e631cadf604246c0cdcc497e718d6949df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Oct 2022 08:51:56 +0100 Subject: [PATCH 007/112] build(deps): bump psycopg2-binary from 2.9.4 to 2.9.5 in /examples/double-proxy (#23658) build(deps): bump psycopg2-binary in /examples/double-proxy Bumps [psycopg2-binary](https://github.com/psycopg/psycopg2) from 2.9.4 to 2.9.5. - [Release notes](https://github.com/psycopg/psycopg2/releases) - [Changelog](https://github.com/psycopg/psycopg2/blob/master/NEWS) - [Commits](https://github.com/psycopg/psycopg2/commits) --- updated-dependencies: - dependency-name: psycopg2-binary dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/double-proxy/requirements.txt | 129 +++++++++++++------------ 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/examples/double-proxy/requirements.txt b/examples/double-proxy/requirements.txt index d914e8caf97c..a5b15f6bada9 100644 --- a/examples/double-proxy/requirements.txt +++ b/examples/double-proxy/requirements.txt @@ -4,64 +4,73 @@ # # pip-compile --allow-unsafe --generate-hashes requirements.in # -psycopg2-binary==2.9.4 \ - --hash=sha256:02cde837df012fa5d579b9cf4bc8e1feb460f38d61f7a4ab4a919d55a9f6eeef \ - --hash=sha256:044b6ab68613de7ea1e63856627deea091bfea09dea5ab4f050b13250fd18cab \ - --hash=sha256:0a9465f0aa36480c8e7614991cbe8ca8aa16b0517c5398a49648ce345e446c19 \ - --hash=sha256:0d8e0c9eec79fe1ae66691e06e3cc714da6fbd77981209bf32fa823c03dbaff8 \ - --hash=sha256:0eae72190be519bf2629062eab7ac8d4ceec5bd132953cefa1596584d86964fe \ - --hash=sha256:15e0ac0ed8a85f6049e836e95ddee627766561c85be8d23f4b3edb6ddbaa7310 \ - --hash=sha256:161dc52a617f0bb610a87d391cb2e77fe65b89ebfbd752f4f3217dde701ea196 \ - --hash=sha256:181ac372a5a5308b4076933601a9b5f0cd139b389b0aa5e164786a2abbcdb978 \ - --hash=sha256:1c22c59ab7d9dc110d409445f111f58556bf699b0548f3fc5176684a29c629c4 \ - --hash=sha256:226f11be577b70a57f4910c0ee28591d4d9fcb3d455e966267179156ae2e0c41 \ - --hash=sha256:24d627ed69e754c48dd142a914124858c600b4108c92546eb0ba822e63c0c6e2 \ - --hash=sha256:2535f44b00f26f6af0e949c825e6aecb9adcb56c965c17af5b97137fb69f00c0 \ - --hash=sha256:25e0517ad7ee3c5c3c69dbe3c1d95504c811e42f452b39a3505d0763b1f6caa0 \ - --hash=sha256:2903bf90b1e6bfc9bbfc94a1db0b50ffa9830a0ca4c042fbc38d93890c02ce08 \ - --hash=sha256:2f1ded23d17af0d738e7e78087f0b88a53228887845b1989b03af4dfd3fef703 \ - --hash=sha256:30200b07779446760813eef06098ec6d084131e4365b4e023eb43100de758b11 \ - --hash=sha256:33ac8b4754e6b6b21f3ee180da169d8526d91aee9408ec1fc573c16ab32b0207 \ - --hash=sha256:34fd249275faa782c3a2016e86ac2330636ac58d731a1580e7d686e3976b9536 \ - --hash=sha256:44f5dc9b4384bafca8429759ce76c8960ffc2b583fcad9e5dfb3e5f4894269e4 \ - --hash=sha256:451550e0bb5889bbabbf92575a6d6eafced941cc28c86be6ae4667f81bf32d67 \ - --hash=sha256:52383e932e6de5595963f9178cf2af7b9e1f3daacf5135b9c0e21aabbc5bf7c4 \ - --hash=sha256:55137faec669c4277c5687c6ce7c1fbc4dece0e2f14256ee808f4a652f0a2170 \ - --hash=sha256:576b9dfbcd154a0e8b5d9dae6316d037450e64a3b31df87dec71d88e2a2d5e5f \ - --hash=sha256:59a3010d566a48b919490a982f6807f68842686941dc12d568e129d9cd7703d6 \ - --hash=sha256:61c6a258469c66412ae8358a0501df6ccb3bb48aa9c43b56624571ff9767f91d \ - --hash=sha256:63edc507f8cbfbb5903adb75bad8a99f9798981c854df9119dbebab2ec3ee0e1 \ - --hash=sha256:65d5f4e70a2d3fbaa1349236968792611088f3f2dccead36c1626e1d183cc327 \ - --hash=sha256:6a1618260a112a9c93504511f0b6254b4402a8c41b7130dc6d4c9e39aff3aa0c \ - --hash=sha256:704f1fcdc5b606b70563ea696c69bda90caee3a2f45ffc9cee60a901b394a79f \ - --hash=sha256:7751b11cd7f6b952b4b5ec5b93b5be9ce20faba786c18c25c354f5d8717a173c \ - --hash=sha256:7ad9d032dc1a31a86ca7b059f43554a049a2bfda8fe32d1492ad25f6686aff03 \ - --hash=sha256:7b01d07006a0ac2216921b69a220b9f0974345d0b1b36efaeabdc7550b1cc4f8 \ - --hash=sha256:7b47643c45e7619788c081d42e1d9d98c7c8a4933010a9967d097cc3c4c29f41 \ - --hash=sha256:80ed219ce6cb21a5b53ead0edf5b56b6d23de4cb95389ac606f47670474f4816 \ - --hash=sha256:82df4a8600999c4c0cb7d6614df1bbdb3c74732f63e79f78487893ffbed3d083 \ - --hash=sha256:8660112e9127a019969a23c878e1b4a419e8a6427f9a9050c19830f152628c8a \ - --hash=sha256:89a86c2b35460700d04b4d6461153ab39ee85af5a5385acac9563a8310e6320a \ - --hash=sha256:8d7bc25729bb6d96b44f49ad78fde0e27a1a867cb205322b7e5f5b49e04d6f1f \ - --hash=sha256:97e4f3d9b17d12e7c00cb1c29c0040044135cd5146838da4274615dbe0baae78 \ - --hash=sha256:a431deb6ffdfa551f7400b3a94fa4b964837e67f49e3c37aa26d90dc75970816 \ - --hash=sha256:a6a2d3d75d8698dee492f4af7ad07606d0734e581edf9e2ce2f74b6fce90f42e \ - --hash=sha256:ae5b41dbf7731b838021923edfbe3b5ccdec84d92d5795f5229c0d08d32509d9 \ - --hash=sha256:aff258af03dda9a990960a53759d10c3a9b936837c71fe2f3b581acd356b9121 \ - --hash=sha256:b216a15e13f6e763db40ac3beb74b588650bc030d10a78fde182b88d273b82b5 \ - --hash=sha256:b23b25b1243576b952689966205ef7d4285688068b966a1ca0e620bcb390d483 \ - --hash=sha256:b896637091cde69d170a89253dde9aee814b25ca204b7e213fd0a6462e666638 \ - --hash=sha256:d5f27b1d1b56470385faa2b2636fcb823e7ac5b5b734e0aa76b14637c66eb3b7 \ - --hash=sha256:d6ba33f39436191ece7ea2b3d0b4dff00af71acd5c6e6f1d6b7563aa7286e9f2 \ - --hash=sha256:d6c5e1df6f427d7a82606cf8f07cf3ba9fb3f366804b01e65f1f00f8df6b54f1 \ - --hash=sha256:e02f77b620ad6b36564fe41980865436912e21a3b1138cdde175cf24afde1bc5 \ - --hash=sha256:e72491d72870c3cb2f0d6f4174485533caec0e9ed7e717e2859b7cc7ff2ae1c4 \ - --hash=sha256:ea8d5cd689fa7225d81ae0a049ba03e0165f4ed9ca083b19a405be9ad0b36845 \ - --hash=sha256:eb5341fc7c53fdd95ac2415be77b1de854ab266488cff71174ebb007baf0e675 \ - --hash=sha256:edf0a66ce9517365c7dcfed597894d8dd1f27b59e550b77a089054101435213b \ - --hash=sha256:f225784812b2b57d340f2eb0d2cebef989dcc82c288f5553e28ee9767c7c8344 \ - --hash=sha256:f5fbb3b325c65010e04af206a9243e2df8606736c510c7f268aca6a93e5294a9 \ - --hash=sha256:f78cafa25731e0b5aa16fe20bea1abf643d4e853f6bfb8a64421b06b878e2b88 \ - --hash=sha256:fb639a0e65dce4a9cccbcbdd8ddd0c8c6ab10bca317b827a5c52ac3c3a4ad60a \ - --hash=sha256:ffb2f288f577a748cc23c65a818290755a4c2da1f87a40d7055b61a096d31e20 +psycopg2-binary==2.9.5 \ + --hash=sha256:00475004e5ed3e3bf5e056d66e5dcdf41a0dc62efcd57997acd9135c40a08a50 \ + --hash=sha256:01ad49d68dd8c5362e4bfb4158f2896dc6e0c02e87b8a3770fc003459f1a4425 \ + --hash=sha256:024030b13bdcbd53d8a93891a2cf07719715724fc9fee40243f3bd78b4264b8f \ + --hash=sha256:02551647542f2bf89073d129c73c05a25c372fc0a49aa50e0de65c3c143d8bd0 \ + --hash=sha256:043a9fd45a03858ff72364b4b75090679bd875ee44df9c0613dc862ca6b98460 \ + --hash=sha256:05b3d479425e047c848b9782cd7aac9c6727ce23181eb9647baf64ffdfc3da41 \ + --hash=sha256:0775d6252ccb22b15da3b5d7adbbf8cfe284916b14b6dc0ff503a23edb01ee85 \ + --hash=sha256:1764546ffeaed4f9428707be61d68972eb5ede81239b46a45843e0071104d0dd \ + --hash=sha256:1e491e6489a6cb1d079df8eaa15957c277fdedb102b6a68cfbf40c4994412fd0 \ + --hash=sha256:212757ffcecb3e1a5338d4e6761bf9c04f750e7d027117e74aa3cd8a75bb6fbd \ + --hash=sha256:215d6bf7e66732a514f47614f828d8c0aaac9a648c46a831955cb103473c7147 \ + --hash=sha256:25382c7d174c679ce6927c16b6fbb68b10e56ee44b1acb40671e02d29f2fce7c \ + --hash=sha256:2d964eb24c8b021623df1c93c626671420c6efadbdb8655cb2bd5e0c6fa422ba \ + --hash=sha256:2ec46ed947801652c9643e0b1dc334cfb2781232e375ba97312c2fc256597632 \ + --hash=sha256:2ef892cabdccefe577088a79580301f09f2a713eb239f4f9f62b2b29cafb0577 \ + --hash=sha256:33e632d0885b95a8b97165899006c40e9ecdc634a529dca7b991eb7de4ece41c \ + --hash=sha256:3520d7af1ebc838cc6084a3281145d5cd5bdd43fdef139e6db5af01b92596cb7 \ + --hash=sha256:3d790f84201c3698d1bfb404c917f36e40531577a6dda02e45ba29b64d539867 \ + --hash=sha256:3fc33295cfccad697a97a76dec3f1e94ad848b7b163c3228c1636977966b51e2 \ + --hash=sha256:422e3d43b47ac20141bc84b3d342eead8d8099a62881a501e97d15f6addabfe9 \ + --hash=sha256:46512486be6fbceef51d7660dec017394ba3e170299d1dc30928cbedebbf103a \ + --hash=sha256:46850a640df62ae940e34a163f72e26aca1f88e2da79148e1862faaac985c302 \ + --hash=sha256:484405b883630f3e74ed32041a87456c5e0e63a8e3429aa93e8714c366d62bd1 \ + --hash=sha256:4e7904d1920c0c89105c0517dc7e3f5c20fb4e56ba9cdef13048db76947f1d79 \ + --hash=sha256:56b2957a145f816726b109ee3d4e6822c23f919a7d91af5a94593723ed667835 \ + --hash=sha256:5c6527c8efa5226a9e787507652dd5ba97b62d29b53c371a85cd13f957fe4d42 \ + --hash=sha256:5cbc554ba47ecca8cd3396ddaca85e1ecfe3e48dd57dc5e415e59551affe568e \ + --hash=sha256:5d28ecdf191db558d0c07d0f16524ee9d67896edf2b7990eea800abeb23ebd61 \ + --hash=sha256:5fc447058d083b8c6ac076fc26b446d44f0145308465d745fba93a28c14c9e32 \ + --hash=sha256:63e318dbe52709ed10d516a356f22a635e07a2e34c68145484ed96a19b0c4c68 \ + --hash=sha256:68d81a2fe184030aa0c5c11e518292e15d342a667184d91e30644c9d533e53e1 \ + --hash=sha256:6e63814ec71db9bdb42905c925639f319c80e7909fb76c3b84edc79dadef8d60 \ + --hash=sha256:6f8a9bcab7b6db2e3dbf65b214dfc795b4c6b3bb3af922901b6a67f7cb47d5f8 \ + --hash=sha256:70831e03bd53702c941da1a1ad36c17d825a24fbb26857b40913d58df82ec18b \ + --hash=sha256:74eddec4537ab1f701a1647214734bc52cee2794df748f6ae5908e00771f180a \ + --hash=sha256:7b3751857da3e224f5629400736a7b11e940b5da5f95fa631d86219a1beaafec \ + --hash=sha256:7cf1d44e710ca3a9ce952bda2855830fe9f9017ed6259e01fcd71ea6287565f5 \ + --hash=sha256:7d07f552d1e412f4b4e64ce386d4c777a41da3b33f7098b6219012ba534fb2c2 \ + --hash=sha256:7d88db096fa19d94f433420eaaf9f3c45382da2dd014b93e4bf3215639047c16 \ + --hash=sha256:7ee3095d02d6f38bd7d9a5358fcc9ea78fcdb7176921528dd709cc63f40184f5 \ + --hash=sha256:902844f9c4fb19b17dfa84d9e2ca053d4a4ba265723d62ea5c9c26b38e0aa1e6 \ + --hash=sha256:937880290775033a743f4836aa253087b85e62784b63fd099ee725d567a48aa1 \ + --hash=sha256:95076399ec3b27a8f7fa1cc9a83417b1c920d55cf7a97f718a94efbb96c7f503 \ + --hash=sha256:9c38d3869238e9d3409239bc05bc27d6b7c99c2a460ea337d2814b35fb4fea1b \ + --hash=sha256:9e32cedc389bcb76d9f24ea8a012b3cb8385ee362ea437e1d012ffaed106c17d \ + --hash=sha256:9ffdc51001136b699f9563b1c74cc1f8c07f66ef7219beb6417a4c8aaa896c28 \ + --hash=sha256:a0adef094c49f242122bb145c3c8af442070dc0e4312db17e49058c1702606d4 \ + --hash=sha256:a36a0e791805aa136e9cbd0ffa040d09adec8610453ee8a753f23481a0057af5 \ + --hash=sha256:a7e518a0911c50f60313cb9e74a169a65b5d293770db4770ebf004245f24b5c5 \ + --hash=sha256:af0516e1711995cb08dc19bbd05bec7dbdebf4185f68870595156718d237df3e \ + --hash=sha256:b8104f709590fff72af801e916817560dbe1698028cd0afe5a52d75ceb1fce5f \ + --hash=sha256:b911dfb727e247340d36ae20c4b9259e4a64013ab9888ccb3cbba69b77fd9636 \ + --hash=sha256:b9a794cef1d9c1772b94a72eec6da144c18e18041d294a9ab47669bc77a80c1d \ + --hash=sha256:b9c33d4aef08dfecbd1736ceab8b7b3c4358bf10a0121483e5cd60d3d308cc64 \ + --hash=sha256:b9d38a4656e4e715d637abdf7296e98d6267df0cc0a8e9a016f8ba07e4aa3eeb \ + --hash=sha256:bcda1c84a1c533c528356da5490d464a139b6e84eb77cc0b432e38c5c6dd7882 \ + --hash=sha256:c15ba5982c177bc4b23a7940c7e4394197e2d6a424a2d282e7c236b66da6d896 \ + --hash=sha256:c5254cbd4f4855e11cebf678c1a848a3042d455a22a4ce61349c36aafd4c2267 \ + --hash=sha256:c5682a45df7d9642eff590abc73157c887a68f016df0a8ad722dcc0f888f56d7 \ + --hash=sha256:c5e65c6ac0ae4bf5bef1667029f81010b6017795dcb817ba5c7b8a8d61fab76f \ + --hash=sha256:d4c7b3a31502184e856df1f7bbb2c3735a05a8ce0ade34c5277e1577738a5c91 \ + --hash=sha256:d892bfa1d023c3781a3cab8dd5af76b626c483484d782e8bd047c180db590e4c \ + --hash=sha256:dbc332beaf8492b5731229a881807cd7b91b50dbbbaf7fe2faf46942eda64a24 \ + --hash=sha256:dc85b3777068ed30aff8242be2813038a929f2084f69e43ef869daddae50f6ee \ + --hash=sha256:e59137cdb970249ae60be2a49774c6dfb015bd0403f05af1fe61862e9626642d \ + --hash=sha256:e67b3c26e9b6d37b370c83aa790bbc121775c57bfb096c2e77eacca25fd0233b \ + --hash=sha256:e72c91bda9880f097c8aa3601a2c0de6c708763ba8128006151f496ca9065935 \ + --hash=sha256:f95b8aca2703d6a30249f83f4fe6a9abf2e627aa892a5caaab2267d56be7ab69 # via -r requirements.in From eeed086814e80f3245a8900f9650ab3cc7369153 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Oct 2022 08:52:06 +0100 Subject: [PATCH 008/112] build(deps): bump golang from `80ede0f` to `01b565b` in /examples/grpc-bridge/server (#23661) build(deps): bump golang in /examples/grpc-bridge/server Bumps golang from `80ede0f` to `01b565b`. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/grpc-bridge/server/Dockerfile b/examples/grpc-bridge/server/Dockerfile index 82e24aa1b1af..c7cfcd4c62bf 100644 --- a/examples/grpc-bridge/server/Dockerfile +++ b/examples/grpc-bridge/server/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.2-bullseye@sha256:80ede0f12980ec4fc580fa651aabff041d46d1255b323fa0b740ecbce9f89256 as builder +FROM golang:1.19.2-bullseye@sha256:01b565b5e745c650b1893c9a1ba6d3966e5ae4a72ec96595c5a7c54e04d96c05 as builder WORKDIR /build From 48c88d36b0981bfd2f091bc1ee52482a42718bda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Oct 2022 08:53:19 +0100 Subject: [PATCH 009/112] build(deps): bump colorama from 0.4.5 to 0.4.6 in /tools/base (#23656) Bumps [colorama](https://github.com/tartley/colorama) from 0.4.5 to 0.4.6. - [Release notes](https://github.com/tartley/colorama/releases) - [Changelog](https://github.com/tartley/colorama/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tartley/colorama/compare/0.4.5...0.4.6) --- updated-dependencies: - dependency-name: colorama dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 151a0b0a0610..2be70c7de659 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -281,9 +281,9 @@ charset-normalizer==2.0.4 \ # via # aiohttp # requests -colorama==0.4.5 \ - --hash=sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da \ - --hash=sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4 +colorama==0.4.6 \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 # via # -r requirements.in # envoy-docs-sphinx-runner From b58fb72476fac20f213c4a4a09a97d709f736442 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 26 Oct 2022 01:07:54 -0700 Subject: [PATCH 010/112] bazel: update brotli (#23668) This fixes https://github.com/envoyproxy/envoy/issues/23570 and https://github.com/envoyproxy/envoy/issues/23342 There haven't been many commits since the previous pinned version, here is the full diff: https://github.com/google/brotli/compare/0cd2e3926e95e7e2930f57ae3f4885508d462a25...6d03dfbedda1615c4cba1211f8d81735575209c8 Signed-off-by: Keith Smiley --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index abe8e24565a1..34c1f7474d96 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -669,8 +669,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_url = "https://brotli.org", # Use the dev branch of brotli to resolve compilation issues. # TODO(rojkov): Remove when brotli > 1.0.9 is released. - version = "0cd2e3926e95e7e2930f57ae3f4885508d462a25", - sha256 = "93810780e60304b51f2c9645fe313a6e4640711063ed0b860cfa60999dd256c5", + version = "6d03dfbedda1615c4cba1211f8d81735575209c8", + sha256 = "0e8eea905081ce894d1616970a83b21265a13505ce06e8aa6a747fd686938d10", strip_prefix = "brotli-{version}", urls = ["https://github.com/google/brotli/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -678,7 +678,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.compression.brotli.compressor", "envoy.compression.brotli.decompressor", ], - release_date = "2020-09-08", + release_date = "2022-10-25", cpe = "cpe:2.3:a:google:brotli:*", license = "MIT", license_url = "https://github.com/google/brotli/blob/{version}/LICENSE", From 3c66af2ee19bd3aeff8fa057357cfce7f23cff60 Mon Sep 17 00:00:00 2001 From: Seki Rocc Date: Wed, 26 Oct 2022 19:49:48 +0800 Subject: [PATCH 011/112] refactor: code sytle consistency (#23388) * refactor: code sytle consistency, use parent api when possible, just like inc/decBy Signed-off-by: Seki Rocc --- source/common/upstream/resource_manager_impl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/upstream/resource_manager_impl.h b/source/common/upstream/resource_manager_impl.h index 3d01d803da77..cd47e2ccf79c 100644 --- a/source/common/upstream/resource_manager_impl.h +++ b/source/common/upstream/resource_manager_impl.h @@ -25,7 +25,6 @@ struct ManagedResourceImpl : public BasicResourceLimitImpl { } // BasicResourceLimitImpl - bool canCreate() override { return current_ < max(); } void inc() override { BasicResourceLimitImpl::inc(); updateRemaining(); From 9f62abda6bd1987620ead497629c7490fb99851e Mon Sep 17 00:00:00 2001 From: Ismo Puustinen Date: Wed, 26 Oct 2022 08:23:37 -0400 Subject: [PATCH 012/112] decompression: deprecate runtime guard append_to_accept_content_encoding_only_once. (#23622) * decompression: deprecate runtime guard. Deprecate envoy_reloadable_features_append_to_accept_content_encoding_only_once runtime guard. Signed-off-by: Ismo Puustinen --- changelogs/current.yaml | 1 + source/common/runtime/runtime_features.cc | 1 - .../http/decompressor/decompressor_filter.cc | 13 +++------- .../filters/http/decompressor/BUILD | 1 - .../decompressor/decompressor_filter_test.cc | 26 +++---------------- 5 files changed, 7 insertions(+), 35 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 02c61a9a48c0..f344abaaa85c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -35,6 +35,7 @@ removed_config_or_runtime: removed ``envoy.reloadable_features.append_or_truncate`` and legacy code paths. - area: http change: | + removed ``envoy.reloadable_features.append_to_accept_content_encoding_only_once`` and legacy code paths. removed ``envoy.reloadable_features.http1_lazy_read_disable`` and legacy code paths. new_features: diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 972d52872051..36832d73f53f 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -33,7 +33,6 @@ RUNTIME_GUARD(envoy_reloadable_features_admin_stats_filter_use_re2); RUNTIME_GUARD(envoy_reloadable_features_allow_concurrency_for_alpn_pool); RUNTIME_GUARD(envoy_reloadable_features_allow_multiple_dns_addresses); RUNTIME_GUARD(envoy_reloadable_features_allow_upstream_filters); -RUNTIME_GUARD(envoy_reloadable_features_append_to_accept_content_encoding_only_once); RUNTIME_GUARD(envoy_reloadable_features_cares_accept_nodata); RUNTIME_GUARD(envoy_reloadable_features_closer_shadow_behavior); RUNTIME_GUARD(envoy_reloadable_features_combine_sds_requests); diff --git a/source/extensions/filters/http/decompressor/decompressor_filter.cc b/source/extensions/filters/http/decompressor/decompressor_filter.cc index f160c88fa7cd..5f4bb2ac6113 100644 --- a/source/extensions/filters/http/decompressor/decompressor_filter.cc +++ b/source/extensions/filters/http/decompressor/decompressor_filter.cc @@ -3,7 +3,6 @@ #include "source/common/buffer/buffer_impl.h" #include "source/common/common/empty_string.h" #include "source/common/common/macros.h" -#include "source/common/runtime/runtime_features.h" namespace Envoy { namespace Extensions { @@ -73,15 +72,9 @@ Http::FilterHeadersStatus DecompressorFilter::decodeHeaders(Http::RequestHeaderM // the upstream that this hop is able to decompress responses via the Accept-Encoding header. if (config_->responseDirectionConfig().decompressionEnabled() && config_->requestDirectionConfig().advertiseAcceptEncoding()) { - if (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.append_to_accept_content_encoding_only_once")) { - const std::string new_accept_encoding_header = - Http::HeaderUtility::addEncodingToAcceptEncoding( - headers.getInlineValue(accept_encoding_handle.handle()), config_->contentEncoding()); - headers.setInline(accept_encoding_handle.handle(), new_accept_encoding_header); - } else { - headers.appendInline(accept_encoding_handle.handle(), config_->contentEncoding(), ","); - } + const std::string new_accept_encoding_header = Http::HeaderUtility::addEncodingToAcceptEncoding( + headers.getInlineValue(accept_encoding_handle.handle()), config_->contentEncoding()); + headers.setInline(accept_encoding_handle.handle(), new_accept_encoding_header); ENVOY_STREAM_LOG(debug, "DecompressorFilter::decodeHeaders advertise Accept-Encoding with value '{}'", diff --git a/test/extensions/filters/http/decompressor/BUILD b/test/extensions/filters/http/decompressor/BUILD index 4168466a7ddc..6ab72ce93c91 100644 --- a/test/extensions/filters/http/decompressor/BUILD +++ b/test/extensions/filters/http/decompressor/BUILD @@ -24,7 +24,6 @@ envoy_extension_cc_test( "//test/mocks/http:http_mocks", "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", - "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/extensions/filters/http/decompressor/v3:pkg_cc_proto", ], diff --git a/test/extensions/filters/http/decompressor/decompressor_filter_test.cc b/test/extensions/filters/http/decompressor/decompressor_filter_test.cc index 0ea46065c220..c5722dc5241b 100644 --- a/test/extensions/filters/http/decompressor/decompressor_filter_test.cc +++ b/test/extensions/filters/http/decompressor/decompressor_filter_test.cc @@ -10,7 +10,6 @@ #include "test/mocks/protobuf/mocks.h" #include "test/mocks/runtime/mocks.h" #include "test/mocks/stats/mocks.h" -#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -206,17 +205,8 @@ class DecompressorFilterTest : public testing::TestWithParam { expectDecompression(decompressor_ptr, end_with_data); } - void testAcceptEncodingFilter(bool legacy, const std::string& original_accept_encoding, + void testAcceptEncodingFilter(const std::string& original_accept_encoding, const std::string& final_accept_encoding) { - TestScopedRuntime scoped_runtime; - if (legacy) { - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.append_to_accept_content_encoding_only_once", "false"}}); - - } else { - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.append_to_accept_content_encoding_only_once", "true"}}); - } setUpFilter(R"EOF( decompressor_library: typed_config: @@ -331,22 +321,12 @@ TEST_P(DecompressorFilterTest, ExplicitlyEnableAdvertiseAcceptEncodingOnlyOnce) // Do not duplicate accept-encoding values. Remove extra accept-encoding values for the // content-type we specify. Also remove q-values from our content-type (if not set, it defaults // to 1.0). Test also whitespace in accept-encoding value string. - testAcceptEncodingFilter(false, "br,mock, mock\t,mock ;q=0.3", "br,mock"); -} - -TEST_P(DecompressorFilterTest, ExplicitlyEnableAdvertiseAcceptEncodingOnlyOnceLegacy) { - // legacy test to avoid a breaking change - testAcceptEncodingFilter(true, "br,mock, mock\t,mock ;q=0.3", "br,mock, mock\t,mock ;q=0.3,mock"); + testAcceptEncodingFilter("br,mock, mock\t,mock ;q=0.3", "br,mock"); } TEST_P(DecompressorFilterTest, ExplicitlyEnableAdvertiseAcceptEncodingRemoveQValue) { // If the accept-encoding header had a q-value, it needs to be removed. - testAcceptEncodingFilter(false, "mock;q=0.6", "mock"); -} - -TEST_P(DecompressorFilterTest, ExplicitlyEnableAdvertiseAcceptEncodingRemoveQValueLegacy) { - // legacy test to avoid a breaking change - testAcceptEncodingFilter(true, "mock;q=0.6", "mock;q=0.6,mock"); + testAcceptEncodingFilter("mock;q=0.6", "mock"); } TEST_P(DecompressorFilterTest, DecompressionDisabled) { From 493cf30f0785f2e2a383ecc2cbca71a8c0a7514d Mon Sep 17 00:00:00 2001 From: Bootsie Heffernan Date: Wed, 26 Oct 2022 15:03:10 -0400 Subject: [PATCH 013/112] Adding a new CustomHeaderValue for grpc-encoding (#23659) Commit Message: Adding a new CustomHeaderValue for grpc-encoding Additional Description: Risk Level: low Testing: none Docs Changes: N/A Release Notes: N/A Platform Specific Features: N/A Signed-off-by: Bootsie Heffernan --- source/common/http/headers.h | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/http/headers.h b/source/common/http/headers.h index 33417738fcb7..da764adefd58 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -73,6 +73,7 @@ class CustomHeaderValues { const LowerCaseString Etag{"etag"}; const LowerCaseString Expires{"expires"}; const LowerCaseString GrpcAcceptEncoding{"grpc-accept-encoding"}; + const LowerCaseString GrpcEncoding{"grpc-encoding"}; const LowerCaseString IfMatch{"if-match"}; const LowerCaseString IfNoneMatch{"if-none-match"}; const LowerCaseString IfModifiedSince{"if-modified-since"}; From a0f0ed8a5827850497b71ab07b4c8a2330ebae10 Mon Sep 17 00:00:00 2001 From: code Date: Thu, 27 Oct 2022 03:14:20 +0800 Subject: [PATCH 014/112] load balancer: update related methods for load balancer extensions (#23474) Signed-off-by: wbpcode --- envoy/upstream/load_balancer.h | 33 +++++++++++++++---- envoy/upstream/upstream.h | 7 ++-- .../common/upstream/cluster_manager_impl.cc | 9 +++-- source/common/upstream/original_dst_cluster.h | 1 + source/common/upstream/thread_aware_lb_impl.h | 2 ++ source/common/upstream/upstream_impl.cc | 14 +++++--- source/common/upstream/upstream_impl.h | 8 ++--- .../extensions/clusters/aggregate/cluster.h | 1 + .../clusters/dynamic_forward_proxy/cluster.h | 1 + .../clusters/redis/redis_cluster_lb.h | 1 + .../upstream/cluster_manager_impl_test.cc | 3 +- .../load_balancers/custom_lb_policy.h | 8 ++--- test/mocks/upstream/cluster_info.h | 3 +- .../upstream/typed_load_balancer_factory.h | 5 ++- 14 files changed, 62 insertions(+), 34 deletions(-) diff --git a/envoy/upstream/load_balancer.h b/envoy/upstream/load_balancer.h index 07d72fc5e3f5..62d3624e472f 100644 --- a/envoy/upstream/load_balancer.h +++ b/envoy/upstream/load_balancer.h @@ -150,6 +150,16 @@ class LoadBalancer { using LoadBalancerPtr = std::unique_ptr; +/** + * Necessary parameters for creating a worker local load balancer. + */ +struct LoadBalancerParams { + // The worker local priority set of the target cluster. + const PrioritySet& priority_set; + // The worker local priority set of the local cluster. + const PrioritySet* local_priority_set{}; +}; + /** * Factory for load balancers. */ @@ -158,9 +168,15 @@ class LoadBalancerFactory { virtual ~LoadBalancerFactory() = default; /** - * @return LoadBalancerPtr a new load balancer. + * @return LoadBalancerPtr a new worker local load balancer. + * TODO(wbpcode): remove this method in the future and used the new method below. */ virtual LoadBalancerPtr create() PURE; + + /** + * @return LoadBalancerPtr a new worker local load balancer. + */ + virtual LoadBalancerPtr create(LoadBalancerParams params) PURE; }; using LoadBalancerFactorySharedPtr = std::shared_ptr; @@ -213,7 +229,7 @@ class ThreadAwareLoadBalancer { using ThreadAwareLoadBalancerPtr = std::unique_ptr; /** - * Factory for (thread-aware) load balancers. To support a load balancing policy of + * Factory config for load balancers. To support a load balancing policy of * LOAD_BALANCING_POLICY_CONFIG, at least one load balancer factory corresponding to a policy in * load_balancing_policy must be registered with Envoy. Envoy will use the first policy for which * it has a registered factory. @@ -224,13 +240,18 @@ class TypedLoadBalancerFactory : public Config::TypedFactory { /** * @return ThreadAwareLoadBalancerPtr a new thread-aware load balancer. + * + * @param cluster_info supplies the cluster info. + * @param priority_set supplies the priority set. + * @param runtime supplies the runtime loader. + * @param random supplies the random generator. + * @param time_source supplies the time source. */ virtual ThreadAwareLoadBalancerPtr - create(const PrioritySet& priority_set, ClusterStats& stats, Stats::Scope& stats_scope, - Runtime::Loader& runtime, Random::RandomGenerator& random, - const ::envoy::config::cluster::v3::LoadBalancingPolicy_Policy& lb_policy) PURE; + create(const ClusterInfo& cluster_info, const PrioritySet& priority_set, Runtime::Loader& runtime, + Random::RandomGenerator& random, TimeSource& time_source) PURE; - std::string category() const override { return "envoy.load_balancers"; } + std::string category() const override { return "envoy.load_balancing_policies"; } }; } // namespace Upstream diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h index 6d7a6045be64..2633fc60f707 100644 --- a/envoy/upstream/upstream.h +++ b/envoy/upstream/upstream.h @@ -873,11 +873,10 @@ class ClusterInfo : public Http::FilterChainFactory { } /** - * @return const envoy::config::cluster::v3::LoadBalancingPolicy_Policy& the load balancing policy - * to use for this cluster. + * @return const ProtobufWkt::Message& the validated load balancing policy configuration to use + * for this cluster. */ - virtual const envoy::config::cluster::v3::LoadBalancingPolicy_Policy& - loadBalancingPolicy() const PURE; + virtual const ProtobufTypes::MessagePtr& loadBalancingPolicy() const PURE; /** * @return the load balancer factory for this cluster if the load balancing type is diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index a03026aa7283..c676fac636f3 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -898,12 +898,11 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust } else if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided) { cluster_entry_it->second->thread_aware_lb_ = std::move(new_cluster_pair.second); } else if (cluster_reference.info()->lbType() == LoadBalancerType::LoadBalancingPolicyConfig) { - const auto& policy = cluster_reference.info()->loadBalancingPolicy(); TypedLoadBalancerFactory* typed_lb_factory = cluster_reference.info()->loadBalancerFactory(); RELEASE_ASSERT(typed_lb_factory != nullptr, "ClusterInfo should contain a valid factory"); cluster_entry_it->second->thread_aware_lb_ = - typed_lb_factory->create(cluster_reference.prioritySet(), cluster_reference.info()->stats(), - cluster_reference.info()->statsScope(), runtime_, random_, policy); + typed_lb_factory->create(*cluster_reference.info(), cluster_reference.prioritySet(), + runtime_, random_, time_source_); } updateClusterCounts(); @@ -1183,7 +1182,7 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::updateHost // If an LB is thread aware, create a new worker local LB on membership changes. if (lb_factory_ != nullptr) { ENVOY_LOG(debug, "re-creating local LB for TLS cluster {}", name); - lb_ = lb_factory_->create(); + lb_ = lb_factory_->create({priority_set_, parent_.local_priority_set_}); } } @@ -1551,7 +1550,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::ClusterEntry( case LoadBalancerType::Maglev: case LoadBalancerType::OriginalDst: { ASSERT(lb_factory_ != nullptr); - lb_ = lb_factory_->create(); + lb_ = lb_factory_->create({priority_set_, parent_.local_priority_set_}); break; } } diff --git a/source/common/upstream/original_dst_cluster.h b/source/common/upstream/original_dst_cluster.h index a6f6c6d45fd7..33193bc01255 100644 --- a/source/common/upstream/original_dst_cluster.h +++ b/source/common/upstream/original_dst_cluster.h @@ -107,6 +107,7 @@ class OriginalDstCluster : public ClusterImplBase { // Upstream::LoadBalancerFactory Upstream::LoadBalancerPtr create() override { return std::make_unique(cluster_); } + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); } const std::shared_ptr cluster_; }; diff --git a/source/common/upstream/thread_aware_lb_impl.h b/source/common/upstream/thread_aware_lb_impl.h index 25ce4ace6c27..985e2921e3e8 100644 --- a/source/common/upstream/thread_aware_lb_impl.h +++ b/source/common/upstream/thread_aware_lb_impl.h @@ -156,6 +156,8 @@ class ThreadAwareLoadBalancerBase : public LoadBalancerBase, public ThreadAwareL // Upstream::LoadBalancerFactory LoadBalancerPtr create() override; + // Ignore the params for the thread-aware LB. + LoadBalancerPtr create(LoadBalancerParams) override { return create(); } ClusterStats& stats_; Random::RandomGenerator& random_; diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 600161b4b904..30bfaceeb627 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1036,7 +1036,7 @@ ClusterInfoImpl::ClusterInfoImpl( // If load_balancing_policy is set we will use it directly, ignoring lb_policy. if (config.has_load_balancing_policy()) { - configureLbPolicies(config); + configureLbPolicies(config, server_context); } else { switch (config.lb_policy()) { PANIC_ON_PROTO_ENUM_SENTINEL_VALUES; @@ -1065,7 +1065,7 @@ ClusterInfoImpl::ClusterInfoImpl( lb_type_ = LoadBalancerType::ClusterProvided; break; case envoy::config::cluster::v3::Cluster::LOAD_BALANCING_POLICY_CONFIG: { - configureLbPolicies(config); + configureLbPolicies(config, server_context); break; } } @@ -1154,7 +1154,8 @@ ClusterInfoImpl::ClusterInfoImpl( } // Configures the load balancer based on config.load_balancing_policy -void ClusterInfoImpl::configureLbPolicies(const envoy::config::cluster::v3::Cluster& config) { +void ClusterInfoImpl::configureLbPolicies(const envoy::config::cluster::v3::Cluster& config, + Server::Configuration::ServerFactoryContext& context) { if (config.has_lb_subset_config()) { throw EnvoyException( fmt::format("cluster: LB policy {} cannot be combined with lb_subset_config", @@ -1178,7 +1179,12 @@ void ClusterInfoImpl::configureLbPolicies(const envoy::config::cluster::v3::Clus Config::Utility::getAndCheckFactory( policy.typed_extension_config(), /*is_optional=*/true); if (factory != nullptr) { - load_balancing_policy_ = policy; + // Load and validate the configuration. + load_balancing_policy_ = factory->createEmptyConfigProto(); + Config::Utility::translateOpaqueConfig(policy.typed_extension_config().typed_config(), + context.messageValidationVisitor(), + *load_balancing_policy_); + load_balancer_factory_ = factory; break; } diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 0634757dde1b..3807d8696145 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -718,8 +718,7 @@ class ClusterInfoImpl : public ClusterInfo, // Upstream::ClusterInfo bool addedViaApi() const override { return added_via_api_; } - const envoy::config::cluster::v3::LoadBalancingPolicy_Policy& - loadBalancingPolicy() const override { + const ProtobufTypes::MessagePtr& loadBalancingPolicy() const override { return load_balancing_policy_; } TypedLoadBalancerFactory* loadBalancerFactory() const override { return load_balancer_factory_; } @@ -751,7 +750,8 @@ class ClusterInfoImpl : public ClusterInfo, const envoy::config::core::v3::HttpProtocolOptions& commonHttpProtocolOptions() const override { return http_protocol_options_->common_http_protocol_options_; } - void configureLbPolicies(const envoy::config::cluster::v3::Cluster& config); + void configureLbPolicies(const envoy::config::cluster::v3::Cluster& config, + Server::Configuration::ServerFactoryContext& context); ProtocolOptionsConfigConstSharedPtr extensionProtocolOptions(const std::string& name) const override; LoadBalancerType lbType() const override { return lb_type_; } @@ -925,7 +925,7 @@ class ClusterInfoImpl : public ClusterInfo, LoadBalancerSubsetInfoImpl lb_subset_; const envoy::config::core::v3::Metadata metadata_; Envoy::Config::TypedMetadataImpl typed_metadata_; - envoy::config::cluster::v3::LoadBalancingPolicy_Policy load_balancing_policy_; + ProtobufTypes::MessagePtr load_balancing_policy_; TypedLoadBalancerFactory* load_balancer_factory_ = nullptr; const envoy::config::cluster::v3::Cluster::CommonLbConfig common_lb_config_; const bool drain_connections_on_host_removal_; diff --git a/source/extensions/clusters/aggregate/cluster.h b/source/extensions/clusters/aggregate/cluster.h index ba2dbe908e33..90a267a606ab 100644 --- a/source/extensions/clusters/aggregate/cluster.h +++ b/source/extensions/clusters/aggregate/cluster.h @@ -147,6 +147,7 @@ struct AggregateLoadBalancerFactory : public Upstream::LoadBalancerFactory { cluster_.info(), cluster_.cluster_manager_, cluster_.runtime_, cluster_.random_, cluster_.clusters_); } + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); } const Cluster& cluster_; }; diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.h b/source/extensions/clusters/dynamic_forward_proxy/cluster.h index 7bb6046fbfdf..1cd876e6b916 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.h +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.h @@ -112,6 +112,7 @@ class Cluster : public Upstream::BaseDynamicClusterImpl, // Upstream::LoadBalancerFactory Upstream::LoadBalancerPtr create() override { return std::make_unique(cluster_); } + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); } private: Cluster& cluster_; diff --git a/source/extensions/clusters/redis/redis_cluster_lb.h b/source/extensions/clusters/redis/redis_cluster_lb.h index 1a311db0dea1..fe2870adb912 100644 --- a/source/extensions/clusters/redis/redis_cluster_lb.h +++ b/source/extensions/clusters/redis/redis_cluster_lb.h @@ -150,6 +150,7 @@ class RedisClusterLoadBalancerFactory : public ClusterSlotUpdateCallBack, // Upstream::LoadBalancerFactory Upstream::LoadBalancerPtr create() override; + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); } private: class RedisShard { diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 77d1909226c0..7a6b9ada3f51 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -1132,8 +1132,7 @@ TEST_F(ClusterManagerImplTest, LbPolicyConfig) { create(parseBootstrapFromV3Yaml(yaml)); const auto& cluster = cluster_manager_->clusters().getCluster("cluster_1"); EXPECT_NE(cluster, absl::nullopt); - EXPECT_EQ(cluster->get().info()->loadBalancingPolicy().typed_extension_config().name(), - "envoy.load_balancers.custom_lb"); + EXPECT_NE(cluster->get().info()->loadBalancingPolicy(), nullptr); } // Verify that if Envoy does not have a factory for any of the load balancing policies specified in diff --git a/test/integration/load_balancers/custom_lb_policy.h b/test/integration/load_balancers/custom_lb_policy.h index cbdcd49ff8a8..e166472248bd 100644 --- a/test/integration/load_balancers/custom_lb_policy.h +++ b/test/integration/load_balancers/custom_lb_policy.h @@ -47,6 +47,7 @@ class ThreadAwareLbImpl : public Upstream::ThreadAwareLoadBalancer { LbFactory(const Upstream::HostSharedPtr& host) : host_(host) {} Upstream::LoadBalancerPtr create() override { return std::make_unique(host_); } + Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); } const Upstream::HostSharedPtr host_; }; @@ -62,10 +63,9 @@ class CustomLbFactory : public Upstream::TypedLoadBalancerFactoryBase { return Envoy::ProtobufTypes::MessagePtr{new ::test::integration::custom_lb::CustomLbConfig()}; } - Upstream::ThreadAwareLoadBalancerPtr - create(const Upstream::PrioritySet&, Upstream::ClusterStats&, Stats::Scope&, Runtime::Loader&, - Random::RandomGenerator&, - const ::envoy::config::cluster::v3::LoadBalancingPolicy_Policy&) override { + Upstream::ThreadAwareLoadBalancerPtr create(const Upstream::ClusterInfo&, + const Upstream::PrioritySet&, Runtime::Loader&, + Random::RandomGenerator&, TimeSource&) override { return std::make_unique(); } }; diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index e421e673f813..bb6fded4f26f 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -125,8 +125,7 @@ class MockClusterInfo : public ClusterInfo { (const)); MOCK_METHOD(ProtocolOptionsConfigConstSharedPtr, extensionProtocolOptions, (const std::string&), (const)); - MOCK_METHOD(const envoy::config::cluster::v3::LoadBalancingPolicy_Policy&, loadBalancingPolicy, - (), (const)); + MOCK_METHOD(const ProtobufTypes::MessagePtr&, loadBalancingPolicy, (), (const)); MOCK_METHOD(TypedLoadBalancerFactory*, loadBalancerFactory, (), (const)); MOCK_METHOD(const envoy::config::cluster::v3::Cluster::CommonLbConfig&, lbConfig, (), (const)); MOCK_METHOD(LoadBalancerType, lbType, (), (const)); diff --git a/test/mocks/upstream/typed_load_balancer_factory.h b/test/mocks/upstream/typed_load_balancer_factory.h index 98a3e2fa6a28..eb8b0cb8d4f3 100644 --- a/test/mocks/upstream/typed_load_balancer_factory.h +++ b/test/mocks/upstream/typed_load_balancer_factory.h @@ -15,9 +15,8 @@ class MockTypedLoadBalancerFactory : public TypedLoadBalancerFactory { // Upstream::TypedLoadBalancerFactory MOCK_METHOD(std::string, name, (), (const)); MOCK_METHOD(ThreadAwareLoadBalancerPtr, create, - (const PrioritySet& priority_set, ClusterStats& stats, Stats::Scope& stats_scope, - Runtime::Loader& runtime, Random::RandomGenerator& random, - const ::envoy::config::cluster::v3::LoadBalancingPolicy_Policy& lb_policy)); + (const ClusterInfo& cluster_info, const PrioritySet& priority_set, + Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source)); ProtobufTypes::MessagePtr createEmptyConfigProto() override { // Using Struct instead of a custom per-filter empty config proto From 3bb051ebf02d12eb1518efd6419ea381464a4f07 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 26 Oct 2022 12:18:59 -0700 Subject: [PATCH 015/112] bazel: remove //tools/... from default compdb (#23677) For the compilation commands json file we only care about C/C++ targets of which there are nearly 0 of in tools/, this avoids hitting any jq rules, which fail to resolve on arm64 linux. Fixes https://github.com/envoyproxy/envoy/issues/23649 Signed-off-by: Keith Smiley --- tools/gen_compilation_database.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/gen_compilation_database.py b/tools/gen_compilation_database.py index 1cb0a02885be..9c18f99daf44 100755 --- a/tools/gen_compilation_database.py +++ b/tools/gen_compilation_database.py @@ -118,12 +118,9 @@ def fix_compilation_database(args, db): parser.add_argument('--exclude_contrib', action='store_true') parser.add_argument('--bazel', default='bazel') parser.add_argument( - 'bazel_targets', - nargs='*', - default=[ + 'bazel_targets', nargs='*', default=[ "//source/...", "//test/...", - "//tools/...", "//contrib/...", ]) args = parser.parse_args() From 9e5f1654c183371f28e78ce66324e754df8b5eaf Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 26 Oct 2022 17:28:23 -0400 Subject: [PATCH 016/112] Stricter boundary checks for Duration when it is converted to std::chrono::milliseconds (#23529) Internally protobuf first converts seconds to nanoseconds before converting to milliseconds. This can overflow if the default bound checks are applied to the seconds value. This can potentially be a breaking change for configuration that erroneously supplied values higher than 9223372036, since Envoy will now be rejecting such config. Before it was accepting it and using the "wrapped around" value. This case is also verified by the test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5988544525893632 corpus, however the asan sanitizer needs to be of a newer version to catch the error. Signed-off-by: Yan Avlasov --- source/common/protobuf/utility.cc | 20 ++++++++++++++++---- test/common/protobuf/utility_test.cc | 7 +++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index 912a794c0b34..573f47c4fe6c 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -919,22 +919,34 @@ ProtobufWkt::Value ValueUtil::listValue(const std::vector& v namespace { -void validateDuration(const ProtobufWkt::Duration& duration) { +void validateDuration(const ProtobufWkt::Duration& duration, int64_t max_seconds_value) { if (duration.seconds() < 0 || duration.nanos() < 0) { throw DurationUtil::OutOfRangeException( fmt::format("Expected positive duration: {}", duration.DebugString())); } - if (duration.nanos() > 999999999 || - duration.seconds() > Protobuf::util::TimeUtil::kDurationMaxSeconds) { + if (duration.nanos() > 999999999 || duration.seconds() > max_seconds_value) { throw DurationUtil::OutOfRangeException( fmt::format("Duration out-of-range: {}", duration.DebugString())); } } +void validateDuration(const ProtobufWkt::Duration& duration) { + validateDuration(duration, Protobuf::util::TimeUtil::kDurationMaxSeconds); +} + +void validateDurationAsMilliseconds(const ProtobufWkt::Duration& duration) { + // Apply stricter max boundary to the `seconds` value to avoid overflow. + // Note that protobuf internally converts to nanoseconds. + // The kMaxInt64Nanoseconds = 9223372036, which is about 300 years. + constexpr int64_t kMaxInt64Nanoseconds = + std::numeric_limits::max() / (1000 * 1000 * 1000); + validateDuration(duration, kMaxInt64Nanoseconds); +} + } // namespace uint64_t DurationUtil::durationToMilliseconds(const ProtobufWkt::Duration& duration) { - validateDuration(duration); + validateDurationAsMilliseconds(duration); return Protobuf::util::TimeUtil::DurationToMilliseconds(duration); } diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index d75302810a46..356107e173e9 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -1672,6 +1672,13 @@ TEST(DurationUtilTest, OutOfRange) { duration.set_seconds(Protobuf::util::TimeUtil::kDurationMaxSeconds + 1); EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), DurationUtil::OutOfRangeException); } + { + ProtobufWkt::Duration duration; + constexpr int64_t kMaxInt64Nanoseconds = + std::numeric_limits::max() / (1000 * 1000 * 1000); + duration.set_seconds(kMaxInt64Nanoseconds + 1); + EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), DurationUtil::OutOfRangeException); + } } // Verify WIP accounting of the file based annotations. This test uses the strict validator to test From 03fc613b01b6165ea429fa19a5d1e82235a3ecde Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 27 Oct 2022 04:25:42 +0100 Subject: [PATCH 017/112] deps: Bump `com_github_curl` -> 7.86.0 (#23688) Fix #23685 Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 34c1f7474d96..ac2576fb5490 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1031,8 +1031,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "curl", project_desc = "Library for transferring data with URLs", project_url = "https://curl.haxx.se", - version = "7.85.0", - sha256 = "78a06f918bd5fde3c4573ef4f9806f56372b32ec1829c9ec474799eeee641c27", + version = "7.86.0", + sha256 = "3dfdd39ba95e18847965cd3051ea6d22586609d9011d91df7bc5521288987a82", strip_prefix = "curl-{version}", urls = ["https://github.com/curl/curl/releases/download/curl-{underscore_version}/curl-{version}.tar.gz"], use_category = ["dataplane_ext", "observability_ext"], @@ -1042,7 +1042,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.grpc_credentials.aws_iam", "envoy.tracers.opencensus", ], - release_date = "2022-08-31", + release_date = "2022-10-26", cpe = "cpe:2.3:a:haxx:libcurl:*", license = "curl", license_url = "https://github.com/curl/curl/blob/curl-{underscore_version}/COPYING", From 82667905a934826576e817a8222b0e1117b79b15 Mon Sep 17 00:00:00 2001 From: Peter Jausovec Date: Wed, 26 Oct 2022 20:28:35 -0700 Subject: [PATCH 018/112] docs: fix a typo in ext_authz docs (#23697) remove an extra dot in docs Signed-off-by: Peter Jausovec --- api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto | 2 +- .../extensions/filters/http/ext_authz/v3/ext_authz.proto | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto index abf7ddd93311..c79dd4a24bc9 100644 --- a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto +++ b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto @@ -194,7 +194,7 @@ message AuthorizationResponse { // Note that coexistent headers will be overridden. type.matcher.ListStringMatcher allowed_upstream_headers = 1; - // When this :ref:`list `. is set, authorization + // When this :ref:`list ` is set, authorization // response headers that have a correspondent match will be added to the client's response. Note // that when this list is *not* set, all the authorization response headers, except *Authority // (Host)* will be in the response to the client. When a header is included in this list, *Path*, diff --git a/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto b/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto index 3f4fd5928a20..149ff5b48c05 100644 --- a/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto +++ b/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto @@ -268,14 +268,14 @@ message AuthorizationResponse { // that coexistent headers will be appended. type.matcher.v3.ListStringMatcher allowed_upstream_headers_to_append = 3; - // When this :ref:`list `. is set, authorization + // When this :ref:`list ` is set, authorization // response headers that have a correspondent match will be added to the client's response. Note // that when this list is *not* set, all the authorization response headers, except ``Authority // (Host)`` will be in the response to the client. When a header is included in this list, ``Path``, // ``Status``, ``Content-Length``, ``WWWAuthenticate`` and ``Location`` are automatically added. type.matcher.v3.ListStringMatcher allowed_client_headers = 2; - // When this :ref:`list `. is set, authorization + // When this :ref:`list ` is set, authorization // response headers that have a correspondent match will be added to the client's response when // the authorization response itself is successful, i.e. not failed or denied. When this list is // *not* set, no additional headers will be added to the client's response on success. From 3f399d3b537f48f86f8bb25957e5ab7baa4fe95b Mon Sep 17 00:00:00 2001 From: Raven Black Date: Thu, 27 Oct 2022 05:55:05 -0700 Subject: [PATCH 019/112] [cache] Add completion callback to updateHeaders (#23666) Commit Message: Add completion callback to updateHeaders in HttpCache Additional Description: This is necessary to facilitate testing of asynchronous cache implementations. Risk Level: Low - cache filter is still tagged WIP. Testing: Covered by existing tests (and future tests that rely on this change.) Docs Changes: n/a Release Notes: May require any existing HttpCache implementations to update to match the new interface. Platform Specific Features: n/a Signed-off-by: Raven Black --- changelogs/current.yaml | 3 + .../filters/http/cache/cache_filter.cc | 3 +- .../filters/http/cache/http_cache.h | 6 +- .../simple_http_cache/simple_http_cache.cc | 6 +- .../simple_http_cache/simple_http_cache.h | 3 +- .../http_cache_implementation_test_common.cc | 57 +++++++++++-------- .../http_cache_implementation_test_common.h | 2 +- 7 files changed, 51 insertions(+), 29 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index f344abaaa85c..4ce68342a1ce 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -5,6 +5,9 @@ behavior_changes: minor_behavior_changes: # *Changes that may cause incompatibilities for some users, but should not for most* +- area: cache_filter + change: | + add a completion callback to updateHeaders interface. Any external cache implementations will need to update to match this new interface. See changes to simple_http_cache in PR#23666 for example. bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* diff --git a/source/extensions/filters/http/cache/cache_filter.cc b/source/extensions/filters/http/cache/cache_filter.cc index 685074e443f2..704c969f4e35 100644 --- a/source/extensions/filters/http/cache/cache_filter.cc +++ b/source/extensions/filters/http/cache/cache_filter.cc @@ -567,7 +567,8 @@ void CacheFilter::processSuccessfulValidation(Http::ResponseHeaderMap& response_ // TODO(yosrym93): else the cached entry should be deleted. // Update metadata associated with the cached response. Right now this is only response_time; const ResponseMetadata metadata = {time_source_.systemTime()}; - cache_.updateHeaders(*lookup_, response_headers, metadata); + cache_.updateHeaders(*lookup_, response_headers, metadata, + [](bool updated ABSL_ATTRIBUTE_UNUSED) {}); insert_status_ = InsertStatus::HeaderUpdate; } diff --git a/source/extensions/filters/http/cache/http_cache.h b/source/extensions/filters/http/cache/http_cache.h index 24875ffaf7df..2a2e9c6e30c3 100644 --- a/source/extensions/filters/http/cache/http_cache.h +++ b/source/extensions/filters/http/cache/http_cache.h @@ -252,9 +252,13 @@ class HttpCache { // // This is called when an expired cache entry is successfully validated, to // update the cache entry. + // + // The on_complete callback is called with true if the update is successful, + // false if the update was not performed. virtual void updateHeaders(const LookupContext& lookup_context, const Http::ResponseHeaderMap& response_headers, - const ResponseMetadata& metadata) PURE; + const ResponseMetadata& metadata, + std::function on_complete) PURE; // Returns statically known information about a cache. virtual CacheInfo cacheInfo() const PURE; diff --git a/source/extensions/filters/http/cache/simple_http_cache/simple_http_cache.cc b/source/extensions/filters/http/cache/simple_http_cache/simple_http_cache.cc index fefb0677a34a..43fb113ec4db 100644 --- a/source/extensions/filters/http/cache/simple_http_cache/simple_http_cache.cc +++ b/source/extensions/filters/http/cache/simple_http_cache/simple_http_cache.cc @@ -143,19 +143,22 @@ const absl::flat_hash_set SimpleHttpCache::headersNotToUp void SimpleHttpCache::updateHeaders(const LookupContext& lookup_context, const Http::ResponseHeaderMap& response_headers, - const ResponseMetadata& metadata) { + const ResponseMetadata& metadata, + std::function on_complete) { const auto& simple_lookup_context = static_cast(lookup_context); const Key& key = simple_lookup_context.request().key(); absl::WriterMutexLock lock(&mutex_); auto iter = map_.find(key); if (iter == map_.end() || !iter->second.response_headers_) { + on_complete(false); return; } auto& entry = iter->second; // TODO(tangsaidi) handle Vary header updates properly if (VaryHeaderUtils::hasVary(*(entry.response_headers_))) { + on_complete(false); return; } @@ -188,6 +191,7 @@ void SimpleHttpCache::updateHeaders(const LookupContext& lookup_context, return Http::HeaderMap::Iterate::Continue; }); entry.metadata_ = metadata; + on_complete(true); } SimpleHttpCache::Entry SimpleHttpCache::lookup(const LookupRequest& request) { diff --git a/source/extensions/filters/http/cache/simple_http_cache/simple_http_cache.h b/source/extensions/filters/http/cache/simple_http_cache/simple_http_cache.h index 5194edcdf0e3..7f4d543997d2 100644 --- a/source/extensions/filters/http/cache/simple_http_cache/simple_http_cache.h +++ b/source/extensions/filters/http/cache/simple_http_cache/simple_http_cache.h @@ -43,7 +43,8 @@ class SimpleHttpCache : public HttpCache, public Singleton::Instance { Http::StreamEncoderFilterCallbacks& callbacks) override; void updateHeaders(const LookupContext& lookup_context, const Http::ResponseHeaderMap& response_headers, - const ResponseMetadata& metadata) override; + const ResponseMetadata& metadata, + std::function on_complete) override; CacheInfo cacheInfo() const override; Entry lookup(const LookupRequest& request); diff --git a/test/extensions/filters/http/cache/http_cache_implementation_test_common.cc b/test/extensions/filters/http/cache/http_cache_implementation_test_common.cc index c5ecc7cc64a5..6e8c96af080d 100644 --- a/test/extensions/filters/http/cache/http_cache_implementation_test_common.cc +++ b/test/extensions/filters/http/cache/http_cache_implementation_test_common.cc @@ -61,11 +61,19 @@ HttpCacheImplementationTest::~HttpCacheImplementationTest() { delegate_->tearDown(); } -void HttpCacheImplementationTest::updateHeaders( +bool HttpCacheImplementationTest::updateHeaders( absl::string_view request_path, const Http::TestResponseHeaderMapImpl& response_headers, const ResponseMetadata& metadata) { LookupContextPtr lookup_context = lookup(request_path); - cache()->updateHeaders(*lookup_context, response_headers, metadata); + auto update_promise = std::make_shared>(); + cache()->updateHeaders(*lookup_context, response_headers, metadata, + [update_promise](bool result) { update_promise->set_value(result); }); + auto update_future = update_promise->get_future(); + if (std::future_status::ready != update_future.wait_for(std::chrono::seconds(5))) { + EXPECT_TRUE(false) << "timed out in updateHeaders " << request_path; + return false; + } + return update_future.get(); } LookupContextPtr HttpCacheImplementationTest::lookup(absl::string_view request_path) { @@ -161,12 +169,15 @@ absl::Status HttpCacheImplementationTest::insert(absl::string_view request_path, Http::ResponseHeaderMapPtr HttpCacheImplementationTest::getHeaders(LookupContext& context) { Http::ResponseHeaderMapPtr response_headers_ptr; - context.getHeaders([&response_headers_ptr](LookupResult&& lookup_result) { + auto headers_promise = std::make_shared>(); + context.getHeaders([headers_promise](LookupResult&& lookup_result) { EXPECT_NE(lookup_result.cache_entry_status_, CacheEntryStatus::Unusable); EXPECT_NE(lookup_result.headers_, nullptr); - response_headers_ptr = move(lookup_result.headers_); + headers_promise->set_value(std::move(lookup_result.headers_)); }); - return response_headers_ptr; + auto future = headers_promise->get_future(); + EXPECT_EQ(std::future_status::ready, future.wait_for(std::chrono::seconds(5))); + return future.get(); } std::string HttpCacheImplementationTest::getBody(LookupContext& context, uint64_t start, @@ -217,11 +228,9 @@ testing::AssertionResult HttpCacheImplementationTest::expectLookupSuccessWithHea if (!lookup_context) { return AssertionFailure() << "Expected nonnull lookup_context"; } - - Http::ResponseHeaderMapPtr actual_headers_ptr = getHeaders(*lookup_context); - if (!TestUtility::headerMapEqualIgnoreOrder(headers, *actual_headers_ptr)) { + if (!TestUtility::headerMapEqualIgnoreOrder(headers, *lookup_result_.headers_)) { return AssertionFailure() << "Expected headers: " << headers - << "\nActual: " << *actual_headers_ptr; + << "\nActual: " << *lookup_result_.headers_; } return AssertionSuccess(); } @@ -266,7 +275,7 @@ TEST_P(HttpCacheImplementationTest, PutGet) { {"cache-control", "public,max-age=3600"}}; const std::string body1("Value"); - ASSERT_THAT(insert(move(name_lookup_context), response_headers, body1), IsOk()); + ASSERT_THAT(insert(std::move(name_lookup_context), response_headers, body1), IsOk()); name_lookup_context = lookup(request_path1); EXPECT_TRUE(expectLookupSuccessWithBodyAndTrailers(name_lookup_context.get(), body1)); @@ -275,7 +284,7 @@ TEST_P(HttpCacheImplementationTest, PutGet) { EXPECT_EQ(CacheEntryStatus::Unusable, lookup_result_.cache_entry_status_); const std::string new_body1("NewValue"); - ASSERT_THAT(insert(move(name_lookup_context), response_headers, new_body1), IsOk()); + ASSERT_THAT(insert(std::move(name_lookup_context), response_headers, new_body1), IsOk()); EXPECT_TRUE(expectLookupSuccessWithBodyAndTrailers(lookup(request_path1).get(), new_body1)); } @@ -490,7 +499,7 @@ TEST_P(HttpCacheImplementationTest, VaryOnDisallowedKey) { LookupContextPtr first_value_vary = lookup(request_path); EXPECT_EQ(CacheEntryStatus::Unusable, lookup_result_.cache_entry_status_); const std::string body("one"); - ASSERT_THAT(insert(move(first_value_vary), response_headers, body), Not(IsOk())); + ASSERT_THAT(insert(std::move(first_value_vary), response_headers, body), Not(IsOk())); first_value_vary = lookup(request_path); EXPECT_EQ(CacheEntryStatus::Unusable, lookup_result_.cache_entry_status_); } @@ -531,7 +540,7 @@ TEST_P(HttpCacheImplementationTest, UpdateHeadersAndMetadata) { {":status", "200"}, {"etag", "\"foo\""}, {"content-length", "4"}}; - updateHeaders(request_path_1, response_headers, {time_system_.systemTime()}); + EXPECT_TRUE(updateHeaders(request_path_1, response_headers, {time_system_.systemTime()})); auto lookup_context = lookup(request_path_1); lookup_context->onDestroy(); @@ -541,7 +550,7 @@ TEST_P(HttpCacheImplementationTest, UpdateHeadersAndMetadata) { } } -TEST_P(HttpCacheImplementationTest, UpdateHeadersForMissingKey) { +TEST_P(HttpCacheImplementationTest, UpdateHeadersForMissingKeyFails) { const std::string request_path_1("/name"); Http::TestResponseHeaderMapImpl response_headers{ {":status", "200"}, @@ -550,7 +559,7 @@ TEST_P(HttpCacheImplementationTest, UpdateHeadersForMissingKey) { {"etag", "\"foo\""}, }; time_system_.advanceTimeWait(Seconds(3601)); - updateHeaders(request_path_1, response_headers, {time_system_.systemTime()}); + EXPECT_FALSE(updateHeaders(request_path_1, response_headers, {time_system_.systemTime()})); lookup(request_path_1); EXPECT_EQ(CacheEntryStatus::Unusable, lookup_result_.cache_entry_status_); } @@ -572,7 +581,6 @@ TEST_P(HttpCacheImplementationTest, UpdateHeadersDisabledForVaryHeaders) { // An age header is inserted by `makeLookUpResult` response_headers_1.setReferenceKey(Http::LowerCaseString("age"), "0"); EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_1)); - // Update the date field in the headers time_system_.advanceTimeWait(Seconds(3600)); const SystemTime time_2 = time_system_.systemTime(); @@ -582,7 +590,7 @@ TEST_P(HttpCacheImplementationTest, UpdateHeadersDisabledForVaryHeaders) { {"cache-control", "public,max-age=3600"}, {"accept", "image/*"}, {"vary", "accept"}}; - updateHeaders(request_path_1, response_headers_2, {time_2}); + EXPECT_FALSE(updateHeaders(request_path_1, response_headers_2, {time_2})); response_headers_1.setReferenceKey(Http::LowerCaseString("age"), "3600"); // the age is still 0 because an entry is considered fresh after validation EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_1)); @@ -619,7 +627,7 @@ TEST_P(HttpCacheImplementationTest, UpdateHeadersSkipEtagHeader) { {"cache-control", "public,max-age=3600"}, {"etag", "0000-0000"}}; - updateHeaders(request_path_1, response_headers_2, {time_2}); + EXPECT_TRUE(updateHeaders(request_path_1, response_headers_2, {time_2})); response_headers_3.setReferenceKey(Http::LowerCaseString("age"), "0"); EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_3)); } @@ -677,7 +685,7 @@ TEST_P(HttpCacheImplementationTest, UpdateHeadersSkipSpecificHeaders) { {"etag", "1111-1111"}, {"link", "; rel=\"preconnect\""}}; - updateHeaders(request_path_1, incoming_response_headers, {time_2}); + EXPECT_TRUE(updateHeaders(request_path_1, incoming_response_headers, {time_2})); EXPECT_TRUE( expectLookupSuccessWithHeaders(lookup(request_path_1).get(), expected_response_headers)); } @@ -717,7 +725,7 @@ TEST_P(HttpCacheImplementationTest, UpdateHeadersWithMultivalue) { {"link", "; rel=\"preconnect\""}, {"link", "; rel=\"preconnect\""}}; - updateHeaders(request_path_1, response_headers_2, {time_2}); + EXPECT_TRUE(updateHeaders(request_path_1, response_headers_2, {time_2})); lookup(request_path_1); response_headers_2.setCopy(Http::LowerCaseString("age"), "0"); EXPECT_THAT(lookup_result_.headers_.get(), HeaderMapEqualIgnoreOrder(&response_headers_2)); @@ -734,7 +742,7 @@ TEST_P(HttpCacheImplementationTest, PutGetWithTrailers) { {"cache-control", "public,max-age=3600"}}; const std::string body1("Value"); Http::TestResponseTrailerMapImpl response_trailers{{"why", "is"}, {"sky", "blue"}}; - ASSERT_THAT(insert(move(name_lookup_context), response_headers, body1, response_trailers), + ASSERT_THAT(insert(std::move(name_lookup_context), response_headers, body1, response_trailers), IsOk()); name_lookup_context = lookup(request_path1); EXPECT_TRUE( @@ -745,8 +753,9 @@ TEST_P(HttpCacheImplementationTest, PutGetWithTrailers) { EXPECT_EQ(CacheEntryStatus::Unusable, lookup_result_.cache_entry_status_); const std::string new_body1("NewValue"); - ASSERT_THAT(insert(move(name_lookup_context), response_headers, new_body1, response_trailers), - IsOk()); + ASSERT_THAT( + insert(std::move(name_lookup_context), response_headers, new_body1, response_trailers), + IsOk()); EXPECT_TRUE(expectLookupSuccessWithBodyAndTrailers(lookup(request_path1).get(), new_body1, response_trailers)); EXPECT_TRUE(lookup_result_.has_trailers_); @@ -762,7 +771,7 @@ TEST_P(HttpCacheImplementationTest, EmptyTrailers) { {"date", formatter_.fromTime(time_system_.systemTime())}, {"cache-control", "public,max-age=3600"}}; const std::string body1("Value"); - ASSERT_THAT(insert(move(name_lookup_context), response_headers, body1), IsOk()); + ASSERT_THAT(insert(std::move(name_lookup_context), response_headers, body1), IsOk()); name_lookup_context = lookup(request_path1); EXPECT_TRUE(expectLookupSuccessWithBodyAndTrailers(name_lookup_context.get(), body1)); EXPECT_FALSE(lookup_result_.has_trailers_); diff --git a/test/extensions/filters/http/cache/http_cache_implementation_test_common.h b/test/extensions/filters/http/cache/http_cache_implementation_test_common.h index 919d13591200..362c7c22ea66 100644 --- a/test/extensions/filters/http/cache/http_cache_implementation_test_common.h +++ b/test/extensions/filters/http/cache/http_cache_implementation_test_common.h @@ -77,7 +77,7 @@ class HttpCacheImplementationTest Http::TestResponseTrailerMapImpl getTrailers(LookupContext& context); - void updateHeaders(absl::string_view request_path, + bool updateHeaders(absl::string_view request_path, const Http::TestResponseHeaderMapImpl& response_headers, const ResponseMetadata& metadata); From c0189d01846069d24cb1e0e63257cc5614488095 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 27 Oct 2022 15:13:10 +0100 Subject: [PATCH 020/112] deps: Bump `flake8` and associated packages (#23698) and update `envoy.code.check` to work with them Signed-off-by: Ryan Northey --- tools/base/requirements.in | 2 +- tools/base/requirements.txt | 36 ++++++++++++++++++------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 4498731c19b0..e6f51104ea3f 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -5,7 +5,7 @@ cffi>=1.15.0 colorama coloredlogs envoy.base.utils>=0.3.7 -envoy.code.check>=0.2.1 +envoy.code.check>=0.3.0 envoy.dependency.check>=0.1.5 envoy.dependency.pip_check>=0.1.2 envoy.distribution.release>=0.0.8 diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 2be70c7de659..368c1873ca49 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -51,9 +51,9 @@ aio-core==0.9.1 \ # envoy-github-abstract # envoy-github-release # envoy-gpg-identity -aio-run-checker==0.5.5 \ - --hash=sha256:2422af633b4851551fbdd721d87d48f3cd5a852a19cb0a411284708834258307 \ - --hash=sha256:e4a5d6c7bf138825ac4bb048cd28747a08eecad6d5bb1aa572ed78f27e14c4e3 +aio-run-checker==0.5.7 \ + --hash=sha256:19ce85bc48800c5e0049430346a92d5b2ae98dda95b12ae5332ba1b28e7450e8 \ + --hash=sha256:5b9c5296c824206aebb10ade175eb07e89497761da6749b6ba09507ce03f64af # via # envoy-code-check # envoy-dependency-check @@ -339,9 +339,9 @@ envoy-base-utils==0.3.7 \ # envoy-docs-sphinx-runner # envoy-github-release # envoy-gpg-sign -envoy-code-check==0.2.2 \ - --hash=sha256:25b4fcee184e4b40ad68b26cbfd0614bbe0a5f980aefc95b5e3f9c6803e68bbb \ - --hash=sha256:3ad2dd3e01ff57bfcc21475c1268e79d065959911bc9fe8bf1d3ea3cb3f5adf1 +envoy-code-check==0.3.0 \ + --hash=sha256:9383b4fcfcb22d6733813b3a75c9e40b1ba62f5a13c11b7e3366c31a27e4057a \ + --hash=sha256:f23069e123770e5b3a4e4e050c01d03f6c87144e09f9a20f488ce6768eda5f4c # via -r requirements.in envoy-dependency-check==0.1.5 \ --hash=sha256:2394626236e7b43c0273d4a0a43e1e13d3990ebde9ebdacea253464000927961 \ @@ -394,9 +394,9 @@ envoy-gpg-sign==0.1.0 \ --hash=sha256:14b8efee80916fa78857198fb98357c9ddfaa9dec34eb8a453bda1a364f4b973 \ --hash=sha256:36bb56534a6c5947c18bd10c43185345f092664de86b49318e2afc6cac7d9158 # via -r requirements.in -flake8==4.0.1 \ - --hash=sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d \ - --hash=sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d +flake8==5.0.4 \ + --hash=sha256:6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db \ + --hash=sha256:7a1cf6b73744f5806ab95e526f6f0d8c01c66d7bbe349562d22dfca20610b248 # via # -r requirements.in # envoy-code-check @@ -581,9 +581,9 @@ markupsafe==2.0.1 \ --hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \ --hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872 # via jinja2 -mccabe==0.6.1 \ - --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ - --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f +mccabe==0.7.0 \ + --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325 \ + --hash=sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e # via flake8 multidict==6.0.2 \ --hash=sha256:0327292e745a880459ef71be14e709aaea2f783f3537588fb4ed09b6c01bca60 \ @@ -742,17 +742,17 @@ protobuf==4.21.6 \ # via # envoy-base-utils # envoy-docs-sphinx-runner -pycodestyle==2.8.0 \ - --hash=sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20 \ - --hash=sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f +pycodestyle==2.9.1 \ + --hash=sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785 \ + --hash=sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b # via flake8 pycparser==2.20 \ --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 # via cffi -pyflakes==2.4.0 \ - --hash=sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c \ - --hash=sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e +pyflakes==2.5.0 \ + --hash=sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2 \ + --hash=sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3 # via flake8 pygithub==1.56 \ --hash=sha256:80c6d85cf0f9418ffeb840fd105840af694c4f17e102970badbaf678251f2a01 \ From cfc2c3eaf7923b6f0f15722cb0a3d82d56329cc8 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 27 Oct 2022 12:24:31 -0400 Subject: [PATCH 021/112] test: adding original dst tests. (#23645) Part of #22583 Signed-off-by: Alyssa Wilk --- .../filters/listener/original_dst/BUILD | 20 +++ .../original_dst_integration_test.cc | 149 ++++++++++++++++++ test/integration/base_integration_test.cc | 6 +- test/integration/base_integration_test.h | 3 +- test/integration/integration_tcp_client.cc | 9 +- test/integration/integration_tcp_client.h | 3 +- 6 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 test/extensions/filters/listener/original_dst/original_dst_integration_test.cc diff --git a/test/extensions/filters/listener/original_dst/BUILD b/test/extensions/filters/listener/original_dst/BUILD index 9a17b57d40f4..392e77d4de57 100644 --- a/test/extensions/filters/listener/original_dst/BUILD +++ b/test/extensions/filters/listener/original_dst/BUILD @@ -1,6 +1,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_fuzz_test", + "envoy_cc_test", "envoy_package", ) @@ -17,3 +18,22 @@ envoy_cc_fuzz_test( "//test/extensions/filters/listener/common/fuzz:listener_filter_fuzzer_lib", ], ) + +envoy_cc_test( + name = "original_dst_integration_test", + srcs = [ + "original_dst_integration_test.cc", + ], + deps = [ + "//source/common/http:header_map_lib", + "//source/extensions/filters/http/buffer:config", + "//source/extensions/filters/listener/original_dst:config", + "//source/extensions/filters/network/tcp_proxy:config", + "//test/integration:http_protocol_integration_lib", + "//test/test_common:logging_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/listener/original_dst/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/filters/listener/original_dst/original_dst_integration_test.cc b/test/extensions/filters/listener/original_dst/original_dst_integration_test.cc new file mode 100644 index 000000000000..2dc3f3932632 --- /dev/null +++ b/test/extensions/filters/listener/original_dst/original_dst_integration_test.cc @@ -0,0 +1,149 @@ +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/event/dispatcher.h" +#include "envoy/extensions/filters/listener/original_dst/v3/original_dst.pb.h" +#include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" +#include "envoy/http/header_map.h" + +#include "source/common/http/headers.h" +#include "source/common/network/utility.h" + +#include "test/common/upstream/utility.h" +#include "test/integration/http_integration.h" +#include "test/integration/http_protocol_integration.h" +#include "test/integration/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +using OriginalDstIntegrationTest = HttpProtocolIntegrationTest; + +ConfigHelper::ConfigModifierFunction setOriginalDstCluster(int port) { + return [port](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() == 1, ""); + auto& cluster = *bootstrap.mutable_static_resources()->mutable_clusters(0); + cluster.set_type(envoy::config::cluster::v3::Cluster::ORIGINAL_DST); + cluster.set_lb_policy(envoy::config::cluster::v3::Cluster::CLUSTER_PROVIDED); + cluster.mutable_original_dst_lb_config()->mutable_upstream_port_override()->set_value(port); + cluster.clear_load_assignment(); + + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + listener->mutable_address()->mutable_socket_address()->set_address("0.0.0.0"); + listener->set_traffic_direction(envoy::config::core::v3::INBOUND); + + auto* listener_filter = listener->add_listener_filters(); + listener_filter->set_name("envoy.filters.listener.original_dst"); + envoy::extensions::filters::listener::original_dst::v3::OriginalDst original_dst; + listener_filter->mutable_typed_config()->PackFrom(original_dst); + }; +} + +INSTANTIATE_TEST_SUITE_P(Protocols, OriginalDstIntegrationTest, + testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( + {Http::CodecType::HTTP1}, {Http::CodecType::HTTP1})), + HttpProtocolIntegrationTest::protocolTestParamsToString); + +TEST_P(OriginalDstIntegrationTest, OriginalDstHttpManyConnections) { + // Windows apparently have this loopback property. + DISABLE_UNDER_WINDOWS; + // Only do this for IPv4 as we can have many 127.0.0.x addresses listening on 0.0.0.0 + if (version_ != Network::Address::IpVersion::v4) { + return; + } + ASSERT(!upstream_tls_); + auto address = Network::Test::getAnyAddress(Network::Address::IpVersion::v4); + fake_upstreams_.emplace_back( + std::make_unique(Network::Test::createRawBufferDownstreamSocketFactory(), + address, configWithType(upstreamProtocol()))); + + config_helper_.addConfigModifier( + setOriginalDstCluster(fake_upstreams_[0]->localAddress()->ip()->port())); + initialize(); + + const int32_t kMaxConnections = 10; + for (int i = 0; i < kMaxConnections; ++i) { + std::string address_str = fmt::format("127.0.0.{}", i + 1); + Network::ClientConnectionPtr connection(dispatcher_->createClientConnection( + Network::Utility::resolveUrl(fmt::format("tcp://{}:{}", address_str, lookupPort("http"))), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr, + nullptr)); + + connection->enableHalfClose(enableHalfClose()); + + codec_client_ = makeHttpConnection(std::move(connection)); + auto response = + sendRequestAndWaitForResponse(default_request_headers_, 0, default_response_headers_, 0); + EXPECT_EQ(address_str, fake_upstream_connection_->connection() + .connectionInfoProvider() + .localAddress() + ->ip() + ->addressAsString()); + ASSERT_TRUE(fake_upstream_connection_->close()); + fake_upstream_connection_.reset(); + codec_client_->close(); + } +} + +class OriginalDstTcpProxyIntegrationTest + : public testing::TestWithParam, + public BaseIntegrationTest { +public: + OriginalDstTcpProxyIntegrationTest() + : BaseIntegrationTest(GetParam(), ConfigHelper::tcpProxyConfig()) { + enableHalfClose(true); + } +}; + +TEST_P(OriginalDstTcpProxyIntegrationTest, TestManyConnections) { + // Windows apparently have this loopback property. + DISABLE_UNDER_WINDOWS; + if (version_ != Network::Address::IpVersion::v4) { + return; + } + + // Create an upstream on 0.0.0.0 + auto address = Network::Test::getAnyAddress(Network::Address::IpVersion::v4); + ASSERT(!upstream_tls_); + fake_upstreams_.emplace_back( + std::make_unique(Network::Test::createRawBufferDownstreamSocketFactory(), + address, configWithType(upstreamProtocol()))); + + config_helper_.addConfigModifier( + setOriginalDstCluster(fake_upstreams_[0]->localAddress()->ip()->port())); + initialize(); + + const int32_t kMaxConnections = 10; + for (int i = 0; i < kMaxConnections; ++i) { + // Set up the connection + std::string address_str = fmt::format("127.0.0.{}", i + 1); + IntegrationTcpClientPtr tcp_client = + makeTcpConnection(lookupPort("listener_0"), nullptr, nullptr, address_str); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + EXPECT_EQ(address_str, fake_upstream_connection->connection() + .connectionInfoProvider() + .localAddress() + ->ip() + ->addressAsString()); + + // Write bidirectional data. + ASSERT_TRUE(fake_upstream_connection->write("hello")); + tcp_client->waitForData("hello"); + ASSERT_TRUE(tcp_client->write("hello")); + ASSERT_TRUE(fake_upstream_connection->waitForData(5)); + + // Close down the connection. + ASSERT_TRUE(fake_upstream_connection->write("", true)); + tcp_client->waitForHalfClose(); + ASSERT_TRUE(tcp_client->write("", true)); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + } + test_server_->waitForCounterGe("cluster_manager.cluster_updated", kMaxConnections); +} + +INSTANTIATE_TEST_SUITE_P(IpVersions, OriginalDstTcpProxyIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +} // namespace Envoy diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 6668a7e82b9e..7ed1469f38a0 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -305,9 +305,11 @@ absl::optional BaseIntegrationTest::waitForNextRawUpstreamConnection( IntegrationTcpClientPtr BaseIntegrationTest::makeTcpConnection(uint32_t port, const Network::ConnectionSocket::OptionsSharedPtr& options, - Network::Address::InstanceConstSharedPtr source_address) { + Network::Address::InstanceConstSharedPtr source_address, + absl::string_view destination_address) { return std::make_unique(*dispatcher_, *mock_buffer_factory_, port, version_, - enableHalfClose(), options, source_address); + enableHalfClose(), options, source_address, + destination_address); } void BaseIntegrationTest::registerPort(const std::string& key, uint32_t port) { diff --git a/test/integration/base_integration_test.h b/test/integration/base_integration_test.h index d2b896564692..42f178017b84 100644 --- a/test/integration/base_integration_test.h +++ b/test/integration/base_integration_test.h @@ -99,7 +99,8 @@ class BaseIntegrationTest : protected Logger::Loggable { makeTcpConnection(uint32_t port, const Network::ConnectionSocket::OptionsSharedPtr& options = nullptr, Network::Address::InstanceConstSharedPtr source_address = - Network::Address::InstanceConstSharedPtr()); + Network::Address::InstanceConstSharedPtr(), + absl::string_view destination_address = ""); // Test-wide port map. void registerPort(const std::string& key, uint32_t port); diff --git a/test/integration/integration_tcp_client.cc b/test/integration/integration_tcp_client.cc index 4f2ce7831947..31e2f8b328ca 100644 --- a/test/integration/integration_tcp_client.cc +++ b/test/integration/integration_tcp_client.cc @@ -38,7 +38,7 @@ IntegrationTcpClient::IntegrationTcpClient( Event::Dispatcher& dispatcher, MockBufferFactory& factory, uint32_t port, Network::Address::IpVersion version, bool enable_half_close, const Network::ConnectionSocket::OptionsSharedPtr& options, - Network::Address::InstanceConstSharedPtr source_address) + Network::Address::InstanceConstSharedPtr source_address, absl::string_view destination_address) : payload_reader_(new WaitForPayloadReader(dispatcher)), callbacks_(new ConnectionCallbacks(*this)) { EXPECT_CALL(factory, createBuffer_(_, _, _)) @@ -55,8 +55,11 @@ IntegrationTcpClient::IntegrationTcpClient( })); connection_ = dispatcher.createClientConnection( - Network::Utility::resolveUrl( - fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(version), port)), + Network::Utility::resolveUrl(fmt::format( + "tcp://{}:{}", + destination_address.empty() ? Network::Test::getLoopbackAddressUrlString(version) + : destination_address, + port)), source_address, Network::Test::createRawBufferSocket(), options, nullptr); ON_CALL(*client_write_buffer_, drain(_)) diff --git a/test/integration/integration_tcp_client.h b/test/integration/integration_tcp_client.h index af75b84de08c..65d8fd1dba97 100644 --- a/test/integration/integration_tcp_client.h +++ b/test/integration/integration_tcp_client.h @@ -29,7 +29,8 @@ class IntegrationTcpClient { IntegrationTcpClient(Event::Dispatcher& dispatcher, MockBufferFactory& factory, uint32_t port, Network::Address::IpVersion version, bool enable_half_close, const Network::ConnectionSocket::OptionsSharedPtr& options, - Network::Address::InstanceConstSharedPtr source_address = nullptr); + Network::Address::InstanceConstSharedPtr source_address = nullptr, + absl::string_view destination_address = ""); void close(); void waitForData(const std::string& data, bool exact_match = true); From 3c645fa924ed02a8d9475f283985e685c9565cf5 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Thu, 27 Oct 2022 10:49:34 -0700 Subject: [PATCH 022/112] build: fix char signed vs unsigned warning (#23675) char defaults to signed on most x86 platforms, but unsigned on arm64. When targeting linux arm64 this produced a warning because character was always >= 0. This change circumvents this warning without changing behavior even though it seems pretty unreasonable to have a character less than 0 in this use case. Theoretically we could disable a warning flag instead but that differs between clang and gcc. Fixes https://github.com/envoyproxy/envoy/issues/23569 Signed-off-by: Keith Smiley Signed-off-by: Keith Smiley --- source/common/common/json_escape_string.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/common/json_escape_string.h b/source/common/common/json_escape_string.h index 679f0ef1da4c..30d2c0684d35 100644 --- a/source/common/common/json_escape_string.h +++ b/source/common/common/json_escape_string.h @@ -64,7 +64,7 @@ class JsonEscaper { position += 2; break; default: - if (character >= 0x00 && character <= 0x1f) { + if (character == 0x00 || (character > 0x00 && character <= 0x1f)) { // Print character as unicode hex. sprintf(&result[position + 1], "u%04x", static_cast(character)); position += 6; @@ -107,7 +107,7 @@ class JsonEscaper { } default: { - if (character >= 0x00 && character <= 0x1f) { + if (character == 0x00 || (character > 0x00 && character <= 0x1f)) { // From character (1 byte) to unicode hex (6 bytes). result += 5; } From 489a41f248ebb02bcba3e30788a6c4ba943b678e Mon Sep 17 00:00:00 2001 From: Adam Kotwasinski Date: Thu, 27 Oct 2022 11:53:09 -0700 Subject: [PATCH 023/112] kafka: upgrade to kafka 3.3.1 (#23354) Signed-off-by: Adam Kotwasinski --- bazel/repository_locations.bzl | 12 +++++----- .../network/source/protocol/generator.py | 22 ++++++++++++++----- .../network_filters/kafka_broker_filter.rst | 2 +- .../network_filters/kafka_mesh_filter.rst | 2 +- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index ac2576fb5490..d57aada18df2 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1205,13 +1205,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Kafka (source)", project_desc = "Open-source distributed event streaming platform", project_url = "https://kafka.apache.org", - version = "3.2.3", - sha256 = "a5b45221e215696769f6ccb741c1e91fa1e18194d7ce02fade797f09efaae03a", + version = "3.3.1", + sha256 = "aab244e6ad1d63a830af1776e1810d355e36900be1e8e4e66b7555af8639e2d2", strip_prefix = "kafka-{version}/clients/src/main/resources/common/message", urls = ["https://github.com/apache/kafka/archive/{version}.zip"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.network.kafka_broker", "envoy.filters.network.kafka_mesh"], - release_date = "2022-09-13", + release_date = "2022-09-29", cpe = "cpe:2.3:a:apache:kafka:*", license = "Apache-2.0", license_url = "https://github.com/apache/kafka/blob/{version}/LICENSE", @@ -1235,11 +1235,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Kafka (server binary)", project_desc = "Open-source distributed event streaming platform", project_url = "https://kafka.apache.org", - version = "3.2.3", - sha256 = "b6f91bc013fcdccd73977d49e20eaebb8fcb121a89a0803d11a9b8f1fc93db80", + version = "3.3.1", + sha256 = "18ad8a365fb111de249d3bb8bf3c96cd1af060ec8fb3e3d1fc4a7ae10d9042de", strip_prefix = "kafka_2.13-{version}", urls = ["https://archive.apache.org/dist/kafka/{version}/kafka_2.13-{version}.tgz"], - release_date = "2022-09-09", + release_date = "2022-10-02", use_category = ["test_only"], ), kafka_python_client = dict( diff --git a/contrib/kafka/filters/network/source/protocol/generator.py b/contrib/kafka/filters/network/source/protocol/generator.py index 08064924f429..8aede752f2a9 100755 --- a/contrib/kafka/filters/network/source/protocol/generator.py +++ b/contrib/kafka/filters/network/source/protocol/generator.py @@ -238,8 +238,8 @@ def parse_field(self, field_spec, highest_possible_version): def parse_type(self, type_name, field_spec, highest_possible_version): """ - Parse a given type element - returns an array type, primitive (e.g. uint32_t) or complex one. - """ + Parse a given type element - returns an array type, primitive (e.g. uint32_t) or complex one. + """ if (type_name.startswith('[]')): # In spec files, array types are defined as `[]underlying_type` instead of having its own # element with type inside. @@ -474,8 +474,8 @@ def is_printable(self): class Primitive(TypeSpecification): """ - Represents a Kafka primitive value. - """ + Represents a Kafka primitive value. + """ USABLE_PRIMITIVE_TYPE_NAMES = [ 'bool', 'int8', 'int16', 'int32', 'int64', 'uint16', 'float64', 'string', 'bytes', @@ -562,7 +562,7 @@ class Primitive(TypeSpecification): def __init__(self, name, custom_default_value): self.original_name = name self.name = Primitive.compute(name, Primitive.KAFKA_TYPE_TO_ENVOY_TYPE) - self.custom_default_value = custom_default_value + self.custom_default_value = Primitive.sanitize_value(self.name, custom_default_value) @staticmethod def compute(name, map): @@ -571,6 +571,18 @@ def compute(name, map): else: raise ValueError(name) + @staticmethod + def sanitize_value(type, arg): + """ + Unfortunately we cannot print Python True/False straight into C++ code, so we lowercase. + """ + if arg is None: + return None + if 'bool' == type: + return str(arg).lower() + else: + return arg + def compute_declaration_chain(self): # Primitives need no declarations. return [] diff --git a/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst b/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst index 81414dd52a53..cc8078bb29a3 100644 --- a/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst +++ b/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst @@ -5,7 +5,7 @@ Kafka Broker filter The Apache Kafka broker filter decodes the client protocol for `Apache Kafka `_, both the requests and responses in the payload. -The message versions in `Kafka 3.2.3 `_ +The message versions in `Kafka 3.3.1 `_ are supported. The filter attempts not to influence the communication between client and brokers, so the messages that could not be decoded (due to Kafka client or broker running a newer version than supported by diff --git a/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst b/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst index c230f59d35f9..75615d8ba26c 100644 --- a/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst +++ b/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst @@ -6,7 +6,7 @@ Kafka Mesh filter The Apache Kafka mesh filter provides a facade for `Apache Kafka `_ producers. Produce requests sent to this filter insance can be forwarded to one of multiple clusters, depending on configured forwarding rules. Corresponding message versions from -Kafka 3.2.3 are supported. +Kafka 3.3.1 are supported. * This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.network.kafka_mesh.v3alpha.KafkaMesh``. * :ref:`v3 API reference ` From 4662b126fa98ff5cd6f109a60bf2e4ba9b984330 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Oct 2022 09:25:22 +0100 Subject: [PATCH 024/112] build(deps): bump actions/dependency-review-action from 2.5.0 to 2.5.1 (#23643) Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 2.5.0 to 2.5.1. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/fd675ced9c17f1393071e1a2e685ab527e585a0c...0efb1d1d84fc9633afcdaad14c485cbbc90ef46c) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/depsreview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml index f17e692e0a3a..266ce59d0e2b 100644 --- a/.github/workflows/depsreview.yml +++ b/.github/workflows/depsreview.yml @@ -9,4 +9,4 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 - name: 'Dependency Review' - uses: actions/dependency-review-action@fd675ced9c17f1393071e1a2e685ab527e585a0c + uses: actions/dependency-review-action@0efb1d1d84fc9633afcdaad14c485cbbc90ef46c From 735ec479adabf5b57332a2676f6f61d728933520 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Oct 2022 09:25:36 +0100 Subject: [PATCH 025/112] build(deps): bump github/codeql-action from 2.1.28 to 2.1.29 (#23695) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.1.28 to 2.1.29. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/cc7986c02bac29104a72998e67239bb5ee2ee110...ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-daily.yml | 4 ++-- .github/workflows/codeql-push.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 16aa254611c1..be3fcf3fc4aa 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -28,7 +28,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@cc7986c02bac29104a72998e67239bb5ee2ee110 + uses: github/codeql-action/init@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -55,4 +55,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@cc7986c02bac29104a72998e67239bb5ee2ee110 + uses: github/codeql-action/analyze@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index 184b8dc123e4..32493d144a8c 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -41,7 +41,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@cc7986c02bac29104a72998e67239bb5ee2ee110 + uses: github/codeql-action/init@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -71,4 +71,4 @@ jobs: - name: Perform CodeQL Analysis if: env.BUILD_TARGETS != '' - uses: github/codeql-action/analyze@cc7986c02bac29104a72998e67239bb5ee2ee110 + uses: github/codeql-action/analyze@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 From a366d4c76127cf93ee32c24f131d2cda2b828a1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Oct 2022 09:26:35 +0100 Subject: [PATCH 026/112] build(deps): bump postgres from `9653626` to `bab8d7b` in /examples/shared/postgres (#23713) build(deps): bump postgres in /examples/shared/postgres Bumps postgres from `9653626` to `bab8d7b`. --- updated-dependencies: - dependency-name: postgres dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/postgres/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/postgres/Dockerfile b/examples/shared/postgres/Dockerfile index d4e28cdec103..daa795e06297 100644 --- a/examples/shared/postgres/Dockerfile +++ b/examples/shared/postgres/Dockerfile @@ -1 +1 @@ -FROM postgres:latest@sha256:9653626520498d747b798ed241d2d52248e2918684164b48ee49302326891a84 +FROM postgres:latest@sha256:bab8d7be6466e029f7fa1e69ff6aa0082704db330572638fd01f2791824774d8 From 2e50e1f5da1e91ece71094023b9ea338d8288624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E6=B3=BD=E8=BD=A9?= Date: Fri, 28 Oct 2022 21:02:21 +0800 Subject: [PATCH 027/112] docs: fix typo in flow_control.md (#23733) Signed-off-by: spacewander --- source/docs/flow_control.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/docs/flow_control.md b/source/docs/flow_control.md index e21b05c8ebba..05b6d4e9c294 100644 --- a/source/docs/flow_control.md +++ b/source/docs/flow_control.md @@ -112,13 +112,13 @@ the following: stream when it's read disabled. There will be a callback scheduled to drain these when the stream is read enabled, it can also be drained if the stream is read enabled, and the stream is invoked with additional data. -* The codec's recieve buffer high watermark is still used in consideration for +* The codec's receive buffer high watermark is still used in consideration for granting peer stream additional window (preserving existing protocol flow - control). The codec's recieve buffer high watermark was rarely used prior as we'd + control). The codec's receive buffer high watermark was rarely used prior as we'd eagerly dispatch data through the filter chain. An exceptions where it could be triggered is if a filter in the filter chain either injects data triggering watermark. - The codec's recieve buffer no longer read disables the stream, as we could be - read disabled elsewhere, buffer data in codec's recieve buffer hitting high + The codec's receive buffer no longer read disables the stream, as we could be + read disabled elsewhere, buffer data in codec's receive buffer hitting high watermark and never switch back to being read enabled. * One side effect of deferring stream processing is the need to defer processing stream close. See ``deferred_stream_close`` in From 9436f59b3b489577ef8486829bbc2dff3ee1b91a Mon Sep 17 00:00:00 2001 From: birenroy Date: Fri, 28 Oct 2022 09:49:29 -0400 Subject: [PATCH 028/112] http: removes the default-true use_new_codec_wrapper runtime feature. (#23627) * Removes the default-true use_new_codec_wrapper runtime feature. Signed-off-by: Biren Roy --- changelogs/current.yaml | 1 + source/common/http/http2/codec_impl.cc | 390 ++++-------------- source/common/http/http2/codec_impl.h | 41 +- source/common/runtime/runtime_features.cc | 2 - test/common/http/codec_impl_fuzz_test.cc | 14 +- test/common/http/http2/codec_impl_test.cc | 213 +++------- test/common/http/http2/codec_impl_test_util.h | 18 +- test/integration/BUILD | 7 - test/integration/fake_upstream.cc | 17 +- .../h2_wrapped_capture_fuzz_test.cc | 32 -- test/integration/http_integration.cc | 6 - test/integration/http_integration.h | 1 - test/integration/http_protocol_integration.cc | 22 +- .../multiplexed_integration_test.cc | 13 +- test/per_file_coverage.sh | 4 +- 15 files changed, 154 insertions(+), 627 deletions(-) delete mode 100644 test/integration/h2_wrapped_capture_fuzz_test.cc diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 4ce68342a1ce..9e0aaab4f80e 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -38,6 +38,7 @@ removed_config_or_runtime: removed ``envoy.reloadable_features.append_or_truncate`` and legacy code paths. - area: http change: | + removed ``envoy.reloadable_features.use_new_codec_wrapper`` and legacy code paths. removed ``envoy.reloadable_features.append_to_accept_content_encoding_only_once`` and legacy code paths. removed ``envoy.reloadable_features.http1_lazy_read_disable`` and legacy code paths. diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 743c775de404..812e3d88f211 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -110,19 +110,7 @@ bool Utility::reconstituteCrumbledCookies(const HeaderString& key, const HeaderS return true; } -nghttp2_session* ProdNghttp2SessionFactory::createOld(const nghttp2_session_callbacks* callbacks, - ConnectionImpl* connection, - const nghttp2_option* options) { - nghttp2_session* session; - nghttp2_session_client_new2(&session, callbacks, connection, options); - return session; -} - -void ProdNghttp2SessionFactory::initOld( - nghttp2_session*, ConnectionImpl* connection, - const envoy::config::core::v3::Http2ProtocolOptions& options) { - connection->sendSettings(options, true); -} +ConnectionImpl::Http2Callbacks ConnectionImpl::http2_callbacks_; std::unique_ptr ProdNghttp2SessionFactory::create(const nghttp2_session_callbacks* callbacks, @@ -361,20 +349,10 @@ void ConnectionImpl::StreamImpl::encodeTrailersBase(const HeaderMap& trailers) { void ConnectionImpl::StreamImpl::encodeMetadata(const MetadataMapVector& metadata_map_vector) { parent_.updateActiveStreamsOnEncode(*this); ASSERT(parent_.allow_metadata_); - if (parent_.use_new_codec_wrapper_) { - NewMetadataEncoder& metadata_encoder = getMetadataEncoder(); - auto sources_vec = metadata_encoder.createSources(metadata_map_vector); - for (auto& source : sources_vec) { - parent_.adapter_->SubmitMetadata(stream_id_, 16 * 1024, std::move(source)); - } - } else { - MetadataEncoder& metadata_encoder = getMetadataEncoderOld(); - if (!metadata_encoder.createPayload(metadata_map_vector)) { - return; - } - for (uint8_t flags : metadata_encoder.payloadFrameFlagBytes()) { - submitMetadata(flags); - } + NewMetadataEncoder& metadata_encoder = getMetadataEncoder(); + auto sources_vec = metadata_encoder.createSources(metadata_map_vector); + for (auto& source : sources_vec) { + parent_.adapter_->SubmitMetadata(stream_id_, 16 * 1024, std::move(source)); } if (parent_.sendPendingFramesAndHandleError()) { @@ -431,11 +409,7 @@ void ConnectionImpl::StreamImpl::processBufferedData() { } void ConnectionImpl::StreamImpl::grantPeerAdditionalStreamWindow() { - if (parent_.use_new_codec_wrapper_) { - parent_.adapter_->MarkDataConsumedForStream(stream_id_, unconsumed_bytes_); - } else { - nghttp2_session_consume(parent_.session_, stream_id_, unconsumed_bytes_); - } + parent_.adapter_->MarkDataConsumedForStream(stream_id_, unconsumed_bytes_); unconsumed_bytes_ = 0; if (parent_.sendPendingFramesAndHandleError()) { // Intended to check through coverage that this error case is tested @@ -660,24 +634,8 @@ void ConnectionImpl::StreamImpl::submitTrailers(const HeaderMap& trailers) { return; } - if (parent_.use_new_codec_wrapper_) { - std::vector final_headers = buildHeaders(trailers); - parent_.adapter_->SubmitTrailer(stream_id_, final_headers); - } else { - std::vector final_headers; - buildHeaders(final_headers, trailers); - int rc = nghttp2_submit_trailer(parent_.session_, stream_id_, final_headers.data(), - final_headers.size()); - ASSERT(rc == 0); - } -} - -void ConnectionImpl::StreamImpl::submitMetadata(uint8_t flags) { - ASSERT(parent_.use_new_codec_wrapper_ == false); - ASSERT(stream_id_ > 0); - const int result = - nghttp2_submit_extension(parent_.session_, METADATA_FRAME_TYPE, flags, stream_id_, nullptr); - ASSERT(result == 0); + std::vector final_headers = buildHeaders(trailers); + parent_.adapter_->SubmitTrailer(stream_id_, final_headers); } ssize_t ConnectionImpl::StreamImpl::onDataSourceRead(uint64_t length, uint32_t* data_flags) { @@ -724,55 +682,39 @@ void ConnectionImpl::StreamImpl::onDataSourceSend(const uint8_t* framehd, size_t void ConnectionImpl::ClientStreamImpl::submitHeaders(const HeaderMap& headers, nghttp2_data_provider* provider) { ASSERT(stream_id_ == -1); - if (parent_.use_new_codec_wrapper_) { - // TODO(birenroy): Once using the new wrapper, migrate callers from nghttp2_data_provider to - // DataFrameSource. - std::unique_ptr data_frame_source; - if (provider) { - data_frame_source = http2::adapter::MakeZeroCopyDataFrameSource( - *provider, &parent_.connection_, - [](nghttp2_session*, nghttp2_frame* frame, const uint8_t* framehd, size_t length, - nghttp2_data_source* source, void*) -> int { - ASSERT(frame->data.padlen == 0); - static_cast(source->ptr)->onDataSourceSend(framehd, length); - return 0; - }); - } - stream_id_ = parent_.adapter_->SubmitRequest(buildHeaders(headers), - std::move(data_frame_source), base()); - } else { - std::vector final_headers; - buildHeaders(final_headers, headers); - stream_id_ = nghttp2_submit_request(parent_.session_, nullptr, final_headers.data(), - final_headers.size(), provider, base()); + // TODO(birenroy): Once using the new wrapper, migrate callers from nghttp2_data_provider to + // DataFrameSource. + std::unique_ptr data_frame_source; + if (provider) { + data_frame_source = http2::adapter::MakeZeroCopyDataFrameSource( + *provider, &parent_.connection_, + [](nghttp2_session*, nghttp2_frame* frame, const uint8_t* framehd, size_t length, + nghttp2_data_source* source, void*) -> int { + ASSERT(frame->data.padlen == 0); + static_cast(source->ptr)->onDataSourceSend(framehd, length); + return 0; + }); } + stream_id_ = + parent_.adapter_->SubmitRequest(buildHeaders(headers), std::move(data_frame_source), base()); ASSERT(stream_id_ > 0); } void ConnectionImpl::ServerStreamImpl::submitHeaders(const HeaderMap& headers, nghttp2_data_provider* provider) { ASSERT(stream_id_ != -1); - if (parent_.use_new_codec_wrapper_) { - std::unique_ptr data_frame_source; - if (provider) { - data_frame_source = http2::adapter::MakeZeroCopyDataFrameSource( - *provider, &parent_.connection_, - [](nghttp2_session*, nghttp2_frame* frame, const uint8_t* framehd, size_t length, - nghttp2_data_source* source, void*) -> int { - ASSERT(frame->data.padlen == 0); - static_cast(source->ptr)->onDataSourceSend(framehd, length); - return 0; - }); - } - parent_.adapter_->SubmitResponse(stream_id_, buildHeaders(headers), - std::move(data_frame_source)); - } else { - std::vector final_headers; - buildHeaders(final_headers, headers); - int rc = nghttp2_submit_response(parent_.session_, stream_id_, final_headers.data(), - final_headers.size(), provider); - ASSERT(rc == 0); + std::unique_ptr data_frame_source; + if (provider) { + data_frame_source = http2::adapter::MakeZeroCopyDataFrameSource( + *provider, &parent_.connection_, + [](nghttp2_session*, nghttp2_frame* frame, const uint8_t* framehd, size_t length, + nghttp2_data_source* source, void*) -> int { + ASSERT(frame->data.padlen == 0); + static_cast(source->ptr)->onDataSourceSend(framehd, length); + return 0; + }); } + parent_.adapter_->SubmitResponse(stream_id_, buildHeaders(headers), std::move(data_frame_source)); } void ConnectionImpl::StreamImpl::onPendingFlushTimer() { @@ -807,13 +749,8 @@ void ConnectionImpl::StreamImpl::encodeDataHelper(Buffer::Instance& data, bool e parent_.stats_.pending_send_bytes_.add(data.length()); pending_send_data_->move(data); if (data_deferred_) { - if (parent_.use_new_codec_wrapper_) { - bool success = parent_.adapter_->ResumeStream(stream_id_); - ASSERT(success); - } else { - int rc = nghttp2_session_resume_data(parent_.session_, stream_id_); - ASSERT(rc == 0); - } + bool success = parent_.adapter_->ResumeStream(stream_id_); + ASSERT(success); data_deferred_ = false; } @@ -886,14 +823,8 @@ void ConnectionImpl::StreamImpl::resetStreamWorker(StreamResetReason reason) { // Handle the case where client streams are reset before headers are created. return; } - if (parent_.use_new_codec_wrapper_) { - parent_.adapter_->SubmitRst(stream_id_, - static_cast(reasonToReset(reason))); - } else { - int rc = nghttp2_submit_rst_stream(parent_.session_, NGHTTP2_FLAG_NONE, stream_id_, - reasonToReset(reason)); - ASSERT(rc == 0); - } + parent_.adapter_->SubmitRst(stream_id_, + static_cast(reasonToReset(reason))); } MetadataEncoder& ConnectionImpl::StreamImpl::getMetadataEncoderOld() { @@ -940,12 +871,10 @@ ConnectionImpl::ConnectionImpl(Network::Connection& connection, CodecStats& stat Random::RandomGenerator& random_generator, const envoy::config::core::v3::Http2ProtocolOptions& http2_options, const uint32_t max_headers_kb, const uint32_t max_headers_count) - : use_new_codec_wrapper_( - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.http2_new_codec_wrapper")), - use_oghttp2_library_( + : use_oghttp2_library_( Runtime::runtimeFeatureEnabled("envoy.reloadable_features.http2_use_oghttp2")), - http2_callbacks_(use_new_codec_wrapper_), stats_(stats), connection_(connection), - max_headers_kb_(max_headers_kb), max_headers_count_(max_headers_count), + stats_(stats), connection_(connection), max_headers_kb_(max_headers_kb), + max_headers_count_(max_headers_count), per_stream_buffer_limit_(http2_options.initial_stream_window_size().value()), stream_error_on_invalid_http_messaging_( http2_options.override_stream_error_on_invalid_http_message().value()), @@ -954,9 +883,6 @@ ConnectionImpl::ConnectionImpl(Network::Connection& connection, CodecStats& stat "envoy.reloadable_features.http2_delay_keepalive_timeout")), random_(random_generator), last_received_data_time_(connection_.dispatcher().timeSource().monotonicTime()) { - // This library can only be used with the wrapper API enabled. - ASSERT(!use_oghttp2_library_ || use_new_codec_wrapper_); - if (http2_options.has_connection_keepalive()) { keepalive_interval_ = std::chrono::milliseconds( PROTOBUF_GET_MS_OR_DEFAULT(http2_options.connection_keepalive(), interval, 0)); @@ -980,9 +906,6 @@ ConnectionImpl::~ConnectionImpl() { for (const auto& stream : active_streams_) { stream->destroy(); } - if (!use_new_codec_wrapper_) { - nghttp2_session_del(session_); - } } void ConnectionImpl::sendKeepalive() { @@ -998,14 +921,7 @@ void ConnectionImpl::sendKeepalive() { std::chrono::duration_cast(now.time_since_epoch()).count(); ENVOY_CONN_LOG(trace, "Sending keepalive PING {}", connection_, ms_since_epoch); - if (use_new_codec_wrapper_) { - adapter_->SubmitPing(ms_since_epoch); - } else { - // The last parameter is an opaque 8-byte buffer, so this cast is safe. - int rc = - nghttp2_submit_ping(session_, 0 /*flags*/, reinterpret_cast(&ms_since_epoch)); - ASSERT(rc == 0); - } + adapter_->SubmitPing(ms_since_epoch); if (sendPendingFramesAndHandleError()) { // Intended to check through coverage that this error case is tested @@ -1062,11 +978,7 @@ Http::Status ConnectionImpl::dispatch(Buffer::Instance& data) { current_slice_ = &slice; dispatching_ = true; ssize_t rc; - if (use_new_codec_wrapper_) { - rc = adapter_->ProcessBytes(absl::string_view(static_cast(slice.mem_), slice.len_)); - } else { - rc = nghttp2_session_mem_recv(session_, static_cast(slice.mem_), slice.len_); - } + rc = adapter_->ProcessBytes(absl::string_view(static_cast(slice.mem_), slice.len_)); if (!nghttp2_callback_status_.ok()) { return nghttp2_callback_status_; } @@ -1100,13 +1012,7 @@ const ConnectionImpl::StreamImpl* ConnectionImpl::getStream(int32_t stream_id) c } ConnectionImpl::StreamImpl* ConnectionImpl::getStreamUnchecked(int32_t stream_id) { - StreamImpl* stream; - if (use_new_codec_wrapper_) { - stream = static_cast(adapter_->GetStreamUserData(stream_id)); - } else { - stream = static_cast(nghttp2_session_get_stream_user_data(session_, stream_id)); - } - return stream; + return static_cast(adapter_->GetStreamUserData(stream_id)); } ConnectionImpl::StreamImpl* ConnectionImpl::getStream(int32_t stream_id) { @@ -1124,11 +1030,7 @@ int ConnectionImpl::onData(int32_t stream_id, const uint8_t* data, size_t len) { // Update the window to the peer unless some consumer of this stream's data has hit a flow control // limit and disabled reads on this stream if (stream->shouldAllowPeerAdditionalStreamWindow()) { - if (use_new_codec_wrapper_) { - adapter_->MarkDataConsumedForStream(stream_id, len); - } else { - nghttp2_session_consume(session_, stream_id, len); - } + adapter_->MarkDataConsumedForStream(stream_id, len); } else { stream->unconsumed_bytes_ += len; } @@ -1136,15 +1038,8 @@ int ConnectionImpl::onData(int32_t stream_id, const uint8_t* data, size_t len) { } void ConnectionImpl::goAway() { - if (use_new_codec_wrapper_) { - adapter_->SubmitGoAway(adapter_->GetHighestReceivedStreamId(), - http2::adapter::Http2ErrorCode::HTTP2_NO_ERROR, ""); - } else { - int rc = nghttp2_submit_goaway(session_, NGHTTP2_FLAG_NONE, - nghttp2_session_get_last_proc_stream_id(session_), - NGHTTP2_NO_ERROR, nullptr, 0); - ASSERT(rc == 0); - } + adapter_->SubmitGoAway(adapter_->GetHighestReceivedStreamId(), + http2::adapter::Http2ErrorCode::HTTP2_NO_ERROR, ""); if (sendPendingFramesAndHandleError()) { // Intended to check through coverage that this error case is tested @@ -1153,12 +1048,7 @@ void ConnectionImpl::goAway() { } void ConnectionImpl::shutdownNotice() { - if (use_new_codec_wrapper_) { - adapter_->SubmitShutdownNotice(); - } else { - int rc = nghttp2_submit_shutdown_notice(session_); - ASSERT(rc == 0); - } + adapter_->SubmitShutdownNotice(); if (sendPendingFramesAndHandleError()) { // Intended to check through coverage that this error case is tested @@ -1167,15 +1057,8 @@ void ConnectionImpl::shutdownNotice() { } Status ConnectionImpl::protocolErrorForTest() { - if (use_new_codec_wrapper_) { - adapter_->SubmitGoAway(adapter_->GetHighestReceivedStreamId(), - http2::adapter::Http2ErrorCode::PROTOCOL_ERROR, ""); - } else { - int rc = nghttp2_submit_goaway(session_, NGHTTP2_FLAG_NONE, - nghttp2_session_get_last_proc_stream_id(session_), - NGHTTP2_PROTOCOL_ERROR, nullptr, 0); - ASSERT(rc == 0); - } + adapter_->SubmitGoAway(adapter_->GetHighestReceivedStreamId(), + http2::adapter::Http2ErrorCode::PROTOCOL_ERROR, ""); return sendPendingFrames(); } @@ -1288,10 +1171,7 @@ Status ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { // It's possible that we are waiting to send a deferred reset, so only raise headers/trailers // if local is not complete. if (!stream->deferred_reset_) { - const bool is_server_session = use_new_codec_wrapper_ - ? adapter_->IsServerSession() - : nghttp2_session_check_server_session(session_); - if (is_server_session || stream->received_noninformational_headers_) { + if (adapter_->IsServerSession() || stream->received_noninformational_headers_) { ASSERT(stream->remote_end_stream_); stream->decodeTrailers(); } else { @@ -1530,15 +1410,9 @@ Status ConnectionImpl::onStreamClose(StreamImpl* stream, uint32_t error_code) { // with outstanding window will contribute to a slow connection-window leak. ENVOY_CONN_LOG(debug, "Recouping {} bytes of flow control window for stream {}.", connection_, stream->unconsumed_bytes_, stream_id); - if (use_new_codec_wrapper_) { - adapter_->MarkDataConsumedForStream(stream_id, stream->unconsumed_bytes_); - stream->unconsumed_bytes_ = 0; - adapter_->SetStreamUserData(stream->stream_id_, nullptr); - } else { - nghttp2_session_consume(session_, stream_id, stream->unconsumed_bytes_); - stream->unconsumed_bytes_ = 0; - nghttp2_session_set_stream_user_data(session_, stream->stream_id_, nullptr); - } + adapter_->MarkDataConsumedForStream(stream_id, stream->unconsumed_bytes_); + stream->unconsumed_bytes_ = 0; + adapter_->SetStreamUserData(stream->stream_id_, nullptr); } return okStatus(); @@ -1573,19 +1447,6 @@ int ConnectionImpl::onMetadataFrameComplete(int32_t stream_id, bool end_metadata return result ? 0 : NGHTTP2_ERR_CALLBACK_FAILURE; } -ssize_t ConnectionImpl::packMetadata(int32_t stream_id, uint8_t* buf, size_t len) { - ASSERT(use_new_codec_wrapper_ == false); - ENVOY_CONN_LOG(trace, "pack METADATA frame on stream {}", connection_, stream_id); - - StreamImpl* stream = getStream(stream_id); - if (stream == nullptr) { - return 0; - } - - MetadataEncoder& encoder = stream->getMetadataEncoderOld(); - return encoder.packNextFramePayload(buf, len); -} - int ConnectionImpl::saveHeader(const nghttp2_frame* frame, HeaderString&& name, HeaderString&& value) { StreamImpl* stream = getStream(frame->hd.stream_id); @@ -1627,12 +1488,7 @@ Status ConnectionImpl::sendPendingFrames() { return okStatus(); } - int rc; - if (use_new_codec_wrapper_) { - rc = adapter_->Send(); - } else { - rc = nghttp2_session_send(session_); - } + const int rc = adapter_->Send(); if (rc != 0) { ASSERT(rc == NGHTTP2_ERR_CALLBACK_FAILURE); return codecProtocolError(nghttp2_strerror(rc)); @@ -1727,64 +1583,9 @@ void ConnectionImpl::sendSettingsHelper( adapter_->SubmitSettings(settings); } -void ConnectionImpl::sendSettingsHelperOld( - const envoy::config::core::v3::Http2ProtocolOptions& http2_options, bool disable_push) { - absl::InlinedVector settings; - auto insertParameter = [&settings](const nghttp2_settings_entry& entry) mutable -> bool { - const auto it = std::find_if(settings.cbegin(), settings.cend(), - [&entry](const nghttp2_settings_entry& existing) { - return entry.settings_id == existing.settings_id; - }); - if (it != settings.end()) { - return false; - } - settings.push_back(entry); - return true; - }; - - // Universally disable receiving push promise frames as we don't currently support - // them. nghttp2 will fail the connection if the other side still sends them. - // TODO(mattklein123): Remove this when we correctly proxy push promise. - // NOTE: This is a special case with respect to custom parameter overrides in that server push is - // not supported and therefore not end user configurable. - if (disable_push) { - settings.push_back( - {static_cast(NGHTTP2_SETTINGS_ENABLE_PUSH), disable_push ? 0U : 1U}); - } - - for (const auto& it : http2_options.custom_settings_parameters()) { - ASSERT(it.identifier().value() <= std::numeric_limits::max()); - const bool result = - insertParameter({static_cast(it.identifier().value()), it.value().value()}); - ASSERT(result); - ENVOY_CONN_LOG(debug, "adding custom settings parameter with id {:#x} to {}", connection_, - it.identifier().value(), it.value().value()); - } - - // Insert named parameters. - settings.insert( - settings.end(), - {{NGHTTP2_SETTINGS_HEADER_TABLE_SIZE, http2_options.hpack_table_size().value()}, - {NGHTTP2_SETTINGS_ENABLE_CONNECT_PROTOCOL, http2_options.allow_connect()}, - {NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, http2_options.max_concurrent_streams().value()}, - {NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, http2_options.initial_stream_window_size().value()}}); - if (!settings.empty()) { - int rc = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, settings.data(), settings.size()); - ASSERT(rc == 0); - } else { - // nghttp2_submit_settings need to be called at least once - int rc = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, nullptr, 0); - ASSERT(rc == 0); - } -} - void ConnectionImpl::sendSettings( const envoy::config::core::v3::Http2ProtocolOptions& http2_options, bool disable_push) { - if (use_new_codec_wrapper_) { - sendSettingsHelper(http2_options, disable_push); - } else { - sendSettingsHelperOld(http2_options, disable_push); - } + sendSettingsHelper(http2_options, disable_push); const uint32_t initial_connection_window_size = http2_options.initial_connection_window_size().value(); @@ -1792,15 +1593,8 @@ void ConnectionImpl::sendSettings( if (initial_connection_window_size != NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE) { ENVOY_CONN_LOG(debug, "updating connection-level initial window size to {}", connection_, initial_connection_window_size); - if (use_new_codec_wrapper_) { - adapter_->SubmitWindowUpdate(0, initial_connection_window_size - - NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE); - } else { - int rc = nghttp2_submit_window_update(session_, NGHTTP2_FLAG_NONE, 0, - initial_connection_window_size - - NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE); - ASSERT(rc == 0); - } + adapter_->SubmitWindowUpdate(0, initial_connection_window_size - + NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE); } } @@ -1842,7 +1636,7 @@ void ConnectionImpl::onUnderlyingConnectionBelowWriteBufferLowWatermark() { } } -ConnectionImpl::Http2Callbacks::Http2Callbacks(bool use_new_codec_wrapper) { +ConnectionImpl::Http2Callbacks::Http2Callbacks() { nghttp2_session_callbacks_new(&callbacks_); nghttp2_session_callbacks_set_send_callback( callbacks_, @@ -1947,18 +1741,6 @@ ConnectionImpl::Http2Callbacks::Http2Callbacks(bool use_new_codec_wrapper) { hd->stream_id, hd->flags == END_METADATA_FLAG); }); - // The new codec does not use the pack_extension callback. - if (!use_new_codec_wrapper) { - nghttp2_session_callbacks_set_pack_extension_callback( - callbacks_, - [](nghttp2_session*, uint8_t* buf, size_t len, const nghttp2_frame* frame, - void* user_data) -> ssize_t { - ASSERT(frame->hd.length <= len); - return static_cast(user_data)->packMetadata(frame->hd.stream_id, buf, - len); - }); - } - nghttp2_session_callbacks_set_error_callback2( callbacks_, [](nghttp2_session*, int, const char* msg, size_t len, void* user_data) -> int { return static_cast(user_data)->onError(absl::string_view(msg, len)); @@ -2157,20 +1939,14 @@ ClientConnectionImpl::ClientConnectionImpl( max_response_headers_count), callbacks_(callbacks) { ClientHttp2Options client_http2_options(http2_options, max_response_headers_kb); - if (use_new_codec_wrapper_) { - if (use_oghttp2_library_) { - adapter_ = http2_session_factory.create(http2_callbacks_.callbacks(), base(), - client_http2_options.ogOptions()); - } else { - adapter_ = http2_session_factory.create(http2_callbacks_.callbacks(), base(), - client_http2_options.options()); - } - http2_session_factory.init(base(), http2_options); + if (use_oghttp2_library_) { + adapter_ = http2_session_factory.create(http2_callbacks_.callbacks(), base(), + client_http2_options.ogOptions()); } else { - session_ = http2_session_factory.createOld(http2_callbacks_.callbacks(), base(), - client_http2_options.options()); - http2_session_factory.initOld(session_, base(), http2_options); + adapter_ = http2_session_factory.create(http2_callbacks_.callbacks(), base(), + client_http2_options.options()); } + http2_session_factory.init(base(), http2_options); allow_metadata_ = http2_options.allow_metadata(); idle_session_requires_ping_interval_ = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT( http2_options.connection_keepalive(), connection_idle_interval, 0)); @@ -2262,25 +2038,20 @@ ServerConnectionImpl::ServerConnectionImpl( callbacks_(callbacks), headers_with_underscores_action_(headers_with_underscores_action) { Http2Options h2_options(http2_options, max_request_headers_kb); - if (use_new_codec_wrapper_) { - auto visitor = std::make_unique( - http2::adapter::Perspective::kServer, *http2_callbacks_.callbacks(), base()); - if (use_oghttp2_library_) { - visitor_ = std::move(visitor); - adapter_ = http2::adapter::OgHttp2Adapter::Create(*visitor_, h2_options.ogOptions()); - } else { - auto adapter = - http2::adapter::NgHttp2Adapter::CreateServerAdapter(*visitor, h2_options.options()); - auto stream_close_listener = [p = adapter.get()](http2::adapter::Http2StreamId stream_id) { - p->RemoveStream(stream_id); - }; - visitor->set_stream_close_listener(std::move(stream_close_listener)); - visitor_ = std::move(visitor); - adapter_ = std::move(adapter); - } + auto visitor = std::make_unique( + http2::adapter::Perspective::kServer, *http2_callbacks_.callbacks(), base()); + if (use_oghttp2_library_) { + visitor_ = std::move(visitor); + adapter_ = http2::adapter::OgHttp2Adapter::Create(*visitor_, h2_options.ogOptions()); } else { - nghttp2_session_server_new2(&session_, http2_callbacks_.callbacks(), base(), - h2_options.options()); + auto adapter = + http2::adapter::NgHttp2Adapter::CreateServerAdapter(*visitor, h2_options.options()); + auto stream_close_listener = [p = adapter.get()](http2::adapter::Http2StreamId stream_id) { + p->RemoveStream(stream_id); + }; + visitor->set_stream_close_listener(std::move(stream_close_listener)); + visitor_ = std::move(visitor); + adapter_ = std::move(adapter); } sendSettings(http2_options, false); allow_metadata_ = http2_options.allow_metadata(); @@ -2308,12 +2079,7 @@ Status ServerConnectionImpl::onBeginHeaders(const nghttp2_frame* frame) { stream->request_decoder_ = &callbacks_.newStream(*stream); stream->stream_id_ = frame->hd.stream_id; LinkedList::moveIntoList(std::move(stream), active_streams_); - if (use_new_codec_wrapper_) { - adapter_->SetStreamUserData(frame->hd.stream_id, active_streams_.front().get()); - } else { - nghttp2_session_set_stream_user_data(session_, frame->hd.stream_id, - active_streams_.front().get()); - } + adapter_->SetStreamUserData(frame->hd.stream_id, active_streams_.front().get()); protocol_constraints_.incrementOpenedStreamCount(); return okStatus(); } diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 3f62402d402c..ed7808c285bf 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -82,15 +82,6 @@ class Http2SessionFactory { using ConnectionImplType = ConnectionImpl; virtual ~Http2SessionFactory() = default; - // Returns a new nghttp2_session to be used with |connection|. - virtual nghttp2_session* createOld(const nghttp2_session_callbacks* callbacks, - ConnectionImplType* connection, - const nghttp2_option* options) PURE; - - // Initializes the |session|. - virtual void initOld(nghttp2_session* session, ConnectionImplType* connection, - const envoy::config::core::v3::Http2ProtocolOptions& options) PURE; - // Returns a new HTTP/2 session to be used with |connection|. virtual std::unique_ptr create(const nghttp2_session_callbacks* callbacks, ConnectionImplType* connection, @@ -108,12 +99,6 @@ class Http2SessionFactory { class ProdNghttp2SessionFactory : public Http2SessionFactory { public: - nghttp2_session* createOld(const nghttp2_session_callbacks* callbacks, ConnectionImpl* connection, - const nghttp2_option* options) override; - - void initOld(nghttp2_session* session, ConnectionImpl* connection, - const envoy::config::core::v3::Http2ProtocolOptions& options) override; - std::unique_ptr create(const nghttp2_session_callbacks* callbacks, ConnectionImpl* connection, const http2::adapter::OgHttp2Adapter::Options& options) override; @@ -155,13 +140,7 @@ class ConnectionImpl : public virtual Connection, Protocol protocol() override { return Protocol::Http2; } void shutdownNotice() override; Status protocolErrorForTest(); // Used in tests to simulate errors. - bool wantsToWrite() override { - if (use_new_codec_wrapper_) { - return adapter_->want_write(); - } else { - return nghttp2_session_want_write(session_); - } - } + bool wantsToWrite() override { return adapter_->want_write(); } // Propagate network connection watermark events to each stream on the connection. void onUnderlyingConnectionAboveWriteBufferHighWatermark() override { for (auto& stream : active_streams_) { @@ -185,7 +164,7 @@ class ConnectionImpl : public virtual Connection, */ class Http2Callbacks { public: - explicit Http2Callbacks(bool use_new_codec_wrapper); + Http2Callbacks(); ~Http2Callbacks(); const nghttp2_session_callbacks* callbacks() { return callbacks_; } @@ -243,8 +222,6 @@ class ConnectionImpl : public virtual Connection, virtual void submitHeaders(const HeaderMap& headers, nghttp2_data_provider* provider) PURE; void encodeTrailersBase(const HeaderMap& headers); void submitTrailers(const HeaderMap& trailers); - // Called iff use_new_codec_wrapper_ is false. - void submitMetadata(uint8_t flags); // Returns true if the stream should defer the local reset stream until after the next call to // sendPendingFrames so pending outbound frames have one final chance to be flushed. If we // submit a reset, nghttp2 will cancel outbound frames that have not yet been sent. @@ -602,14 +579,9 @@ class ConnectionImpl : public virtual Connection, void scheduleProtocolConstraintViolationCallback(); void onProtocolConstraintViolation(); - // Uses a new wrapper API around the underlying HTTP/2 codec. Guarded by the - // "envoy.reloadable_features.http2_new_codec_wrapper" runtime feature flag. - const bool use_new_codec_wrapper_; - // Whether to use the new HTTP/2 library. Only has an effect if `use_new_codec_wrapper` is true. + // Whether to use the new HTTP/2 library. const bool use_oghttp2_library_; - // TODO(birenroy): Make this static again when removing - // use_new_codec_wrapper_. - Http2Callbacks http2_callbacks_; + static Http2Callbacks http2_callbacks_; // If deferred processing, the streams will be in LRU order based on when the // stream encoded to the http2 connection. The LRU property is used when @@ -620,9 +592,6 @@ class ConnectionImpl : public virtual Connection, // Tracks the stream id of the current stream we're processing. // This should only be set while we're in the context of dispatching to nghttp2. absl::optional current_stream_id_; - // Used iff use_new_codec_wrapper_ is false. - nghttp2_session* session_{}; - // Used iff use_new_codec_wrapper_ is true. std::unique_ptr visitor_; std::unique_ptr adapter_; @@ -688,8 +657,6 @@ class ConnectionImpl : public virtual Connection, Status onStreamClose(StreamImpl* stream, uint32_t error_code); int onMetadataReceived(int32_t stream_id, const uint8_t* data, size_t len); int onMetadataFrameComplete(int32_t stream_id, bool end_metadata); - // Called iff use_new_codec_wrapper_ is false. - ssize_t packMetadata(int32_t stream_id, uint8_t* buf, size_t len); // Adds buffer fragment for a new outbound frame to the supplied Buffer::OwnedImpl. void addOutboundFrameFragment(Buffer::OwnedImpl& output, const uint8_t* data, size_t length); diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 36832d73f53f..8084e5db25df 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -46,7 +46,6 @@ RUNTIME_GUARD(envoy_reloadable_features_enable_compression_bomb_protection); RUNTIME_GUARD(envoy_reloadable_features_fix_hash_key); RUNTIME_GUARD(envoy_reloadable_features_get_route_config_factory_by_type); RUNTIME_GUARD(envoy_reloadable_features_http2_delay_keepalive_timeout); -RUNTIME_GUARD(envoy_reloadable_features_http2_new_codec_wrapper); RUNTIME_GUARD(envoy_reloadable_features_http3_sends_early_data); RUNTIME_GUARD(envoy_reloadable_features_http_100_continue_case_insensitive); RUNTIME_GUARD(envoy_reloadable_features_http_reject_path_with_fragment); @@ -86,7 +85,6 @@ FALSE_RUNTIME_GUARD(envoy_reloadable_features_defer_processing_backedup_streams) // TODO(rgs1): Make this enabled after Pinterest tests FALSE_RUNTIME_GUARD(envoy_reloadable_features_thrift_connection_draining); // TODO(birenroy) flip after a burn-in period -// Requires envoy_reloadable_features_http2_new_codec_wrapper to be enabled. FALSE_RUNTIME_GUARD(envoy_reloadable_features_http2_use_oghttp2); // TODO(bencebeky): Finish BalsaParser implementation, then enable by default. See issue #21245. FALSE_RUNTIME_GUARD(envoy_reloadable_features_http1_use_balsa_parser); diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index bb3299289aef..be9982f4e9cc 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -541,7 +541,7 @@ using HttpStreamPtr = std::unique_ptr; namespace { -enum class HttpVersion { Http1, Http2Nghttp2, Http2WrappedNghttp2, Http2Oghttp2 }; +enum class HttpVersion { Http1, Http2Nghttp2, Http2Oghttp2 }; void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersion http_version) { Stats::IsolatedStoreImpl stats_store; @@ -575,17 +575,10 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi break; case HttpVersion::Http2Nghttp2: http2 = true; - scoped_runtime.mergeValues({{"envoy.reloadable_features.http2_new_codec_wrapper", "false"}}); - scoped_runtime.mergeValues({{"envoy.reloadable_features.http2_use_oghttp2", "false"}}); - break; - case HttpVersion::Http2WrappedNghttp2: - http2 = true; - scoped_runtime.mergeValues({{"envoy.reloadable_features.http2_new_codec_wrapper", "true"}}); scoped_runtime.mergeValues({{"envoy.reloadable_features.http2_use_oghttp2", "false"}}); break; case HttpVersion::Http2Oghttp2: http2 = true; - scoped_runtime.mergeValues({{"envoy.reloadable_features.http2_new_codec_wrapper", "true"}}); scoped_runtime.mergeValues({{"envoy.reloadable_features.http2_use_oghttp2", "true"}}); break; } @@ -791,10 +784,6 @@ void codecFuzzHttp2Nghttp2(const test::common::http::CodecImplFuzzTestCase& inpu codecFuzz(input, HttpVersion::Http2Nghttp2); } -void codecFuzzHttp2WrappedNghttp2(const test::common::http::CodecImplFuzzTestCase& input) { - codecFuzz(input, HttpVersion::Http2WrappedNghttp2); -} - void codecFuzzHttp2Oghttp2(const test::common::http::CodecImplFuzzTestCase& input) { codecFuzz(input, HttpVersion::Http2Oghttp2); } @@ -814,7 +803,6 @@ DEFINE_PROTO_FUZZER(const test::common::http::CodecImplFuzzTestCase& input) { // We wrap the calls to *codecFuzz* through these functions in order for // the codec name to explicitly be in any stacktrace. codecFuzzHttp2Nghttp2(input); - codecFuzzHttp2WrappedNghttp2(input); // Prevent oghttp2 from aborting the program. // If when disabling the FATAL log abort the fuzzer will create a test that reaches an // inconsistent state (and crashes/accesses inconsistent memory), then it will be a bug we'll diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 11963b0949db..7c9dadf38579 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -52,7 +52,6 @@ namespace Http2 { enum class Http2Impl { Nghttp2, - WrappedNghttp2, Oghttp2, }; @@ -162,15 +161,9 @@ class Http2CodecImplTestFixture { void setupHttp2Overrides() { switch (http2_implementation_) { case Http2Impl::Nghttp2: - scoped_runtime_.mergeValues({{"envoy.reloadable_features.http2_new_codec_wrapper", "false"}}); - scoped_runtime_.mergeValues({{"envoy.reloadable_features.http2_use_oghttp2", "false"}}); - break; - case Http2Impl::WrappedNghttp2: - scoped_runtime_.mergeValues({{"envoy.reloadable_features.http2_new_codec_wrapper", "true"}}); scoped_runtime_.mergeValues({{"envoy.reloadable_features.http2_use_oghttp2", "false"}}); break; case Http2Impl::Oghttp2: - scoped_runtime_.mergeValues({{"envoy.reloadable_features.http2_new_codec_wrapper", "true"}}); scoped_runtime_.mergeValues({{"envoy.reloadable_features.http2_use_oghttp2", "true"}}); break; } @@ -278,81 +271,43 @@ class Http2CodecImplTestFixture { template uint32_t getStreamReceiveWindowLimit(std::unique_ptr& connection, int32_t stream_id) { - if (http2_implementation_ != Http2Impl::Nghttp2) { - return connection->adapter()->GetStreamReceiveWindowLimit(stream_id); - } else { - return nghttp2_session_get_stream_effective_local_window_size(connection->session(), - stream_id); - } + return connection->adapter()->GetStreamReceiveWindowLimit(stream_id); } template uint32_t getStreamReceiveWindowSize(std::unique_ptr& connection, int32_t stream_id) { - if (http2_implementation_ != Http2Impl::Nghttp2) { - return connection->adapter()->GetStreamReceiveWindowSize(stream_id); - } else { - return nghttp2_session_get_stream_local_window_size(connection->session(), stream_id); - } + return connection->adapter()->GetStreamReceiveWindowSize(stream_id); } template uint32_t getStreamSendWindowSize(std::unique_ptr& connection, int32_t stream_id) { - if (http2_implementation_ != Http2Impl::Nghttp2) { - return connection->adapter()->GetStreamSendWindowSize(stream_id); - } else { - return nghttp2_session_get_stream_remote_window_size(connection->session(), stream_id); - } + return connection->adapter()->GetStreamSendWindowSize(stream_id); } template uint32_t getSendWindowSize(std::unique_ptr& connection) { - if (http2_implementation_ != Http2Impl::Nghttp2) { - return connection->adapter()->GetSendWindowSize(); - } else { - return nghttp2_session_get_remote_window_size(connection->session()); - } + return connection->adapter()->GetSendWindowSize(); } template void submitSettings(std::unique_ptr& connection, const std::list>& settings_values) { - if (http2_implementation_ != Http2Impl::Nghttp2) { - std::vector settings; - for (const auto& setting_pair : settings_values) { - settings.push_back({setting_pair.first, setting_pair.second}); - } - connection->adapter()->SubmitSettings(settings); - } else { - std::vector settings; - for (const auto& setting_pair : settings_values) { - settings.push_back({static_cast(setting_pair.first), setting_pair.second}); - } - EXPECT_EQ(0, nghttp2_submit_settings(connection->session(), NGHTTP2_FLAG_NONE, - settings.data(), settings.size())); + std::vector settings; + for (const auto& setting_pair : settings_values) { + settings.push_back({setting_pair.first, setting_pair.second}); } + connection->adapter()->SubmitSettings(settings); } template int getHpackEncoderDynamicTableSize(std::unique_ptr& connection) { - if (http2_implementation_ != Http2Impl::Nghttp2) { - return connection->adapter()->GetHpackEncoderDynamicTableSize(); - } else { - return nghttp2_session_get_hd_deflate_dynamic_table_size(connection->session()); - } + return connection->adapter()->GetHpackEncoderDynamicTableSize(); } template int getHpackDecoderDynamicTableSize(std::unique_ptr& connection) { - if (http2_implementation_ != Http2Impl::Nghttp2) { - return connection->adapter()->GetHpackDecoderDynamicTableSize(); - } else { - return nghttp2_session_get_hd_inflate_dynamic_table_size(connection->session()); - } + return connection->adapter()->GetHpackDecoderDynamicTableSize(); } template void submitPing(std::unique_ptr& connection, uint32_t ping_id) { - if (http2_implementation_ != Http2Impl::Nghttp2) { - connection->adapter()->SubmitPing(ping_id); - } else { - EXPECT_EQ(0, nghttp2_submit_ping(connection->session(), NGHTTP2_FLAG_NONE, nullptr)); - } + connection->adapter()->SubmitPing(ping_id); } void driveClient() { client_wrapper_->driveDispatch(); } @@ -442,16 +397,11 @@ class Http2CodecImplTest : public ::testing::TestWithParamencodeHeaders(request_headers, false).ok()); driveToCompletion(); - nghttp2_priority_spec spec = {0, 10, 0}; // HTTP/2 codec adds 1 to the number of active streams when computing PRIORITY frames limit constexpr uint32_t max_allowed = 2 * CommonUtility::OptionsLimits::DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM; for (uint32_t i = 0; i < max_allowed + 1; ++i) { - if (http2_implementation_ != Http2Impl::Nghttp2) { - client_->adapter()->SubmitPriorityForStream(1, 0, 10, false); - } else { - EXPECT_EQ(0, nghttp2_submit_priority(client_->session(), NGHTTP2_FLAG_NONE, 1, &spec)); - } + client_->adapter()->SubmitPriorityForStream(1, 0, 10, false); } } @@ -480,11 +430,7 @@ class Http2CodecImplTest : public ::testing::TestWithParamadapter()->SubmitWindowUpdate(1, 1); - } else { - EXPECT_EQ(0, nghttp2_submit_window_update(client_->session(), NGHTTP2_FLAG_NONE, 1, 1)); - } + client_->adapter()->SubmitWindowUpdate(1, 1); } } @@ -561,7 +507,7 @@ TEST_P(Http2CodecImplTest, SimpleRequestResponse) { EXPECT_TRUE(client_wrapper_->status_.ok()); EXPECT_TRUE(server_wrapper_->status_.ok()); - if (http2_implementation_ == Http2Impl::WrappedNghttp2) { + if (http2_implementation_ == Http2Impl::Nghttp2) { // Regression test for issue #19761. EXPECT_EQ(0, getClientDataSourcesSize()); EXPECT_EQ(0, getServerDataSourcesSize()); @@ -2505,22 +2451,18 @@ TEST_P(Http2CodecImplStreamLimitTest, LazyDecreaseMaxConcurrentStreamsConsumeErr ::testing::Values(CommonUtility::OptionsLimits::MIN_INITIAL_CONNECTION_WINDOW_SIZE)) // Deferred reset tests use only small windows so that we can test certain conditions. -INSTANTIATE_TEST_SUITE_P(Http2CodecImplDeferredResetTest, Http2CodecImplDeferredResetTest, - ::testing::Combine(HTTP2SETTINGS_SMALL_WINDOW_COMBINE, - HTTP2SETTINGS_SMALL_WINDOW_COMBINE, - ::testing::Values(Http2Impl::Nghttp2, - Http2Impl::WrappedNghttp2, - Http2Impl::Oghttp2), - ::testing::Bool())); +INSTANTIATE_TEST_SUITE_P( + Http2CodecImplDeferredResetTest, Http2CodecImplDeferredResetTest, + ::testing::Combine(HTTP2SETTINGS_SMALL_WINDOW_COMBINE, HTTP2SETTINGS_SMALL_WINDOW_COMBINE, + ::testing::Values(Http2Impl::Nghttp2, Http2Impl::Oghttp2), + ::testing::Bool())); // Flow control tests only use only small windows so that we can test certain conditions. -INSTANTIATE_TEST_SUITE_P(Http2CodecImplFlowControlTest, Http2CodecImplFlowControlTest, - ::testing::Combine(HTTP2SETTINGS_SMALL_WINDOW_COMBINE, - HTTP2SETTINGS_SMALL_WINDOW_COMBINE, - ::testing::Values(Http2Impl::Nghttp2, - Http2Impl::WrappedNghttp2, - Http2Impl::Oghttp2), - ::testing::Bool())); +INSTANTIATE_TEST_SUITE_P( + Http2CodecImplFlowControlTest, Http2CodecImplFlowControlTest, + ::testing::Combine(HTTP2SETTINGS_SMALL_WINDOW_COMBINE, HTTP2SETTINGS_SMALL_WINDOW_COMBINE, + ::testing::Values(Http2Impl::Nghttp2, Http2Impl::Oghttp2), + ::testing::Bool())); // we separate default/edge cases here to avoid combinatorial explosion #define HTTP2SETTINGS_DEFAULT_COMBINE \ @@ -2532,21 +2474,17 @@ INSTANTIATE_TEST_SUITE_P(Http2CodecImplFlowControlTest, Http2CodecImplFlowContro // Stream limit test only uses the default values because not all combinations of // edge settings allow for the number of streams needed by the test. -INSTANTIATE_TEST_SUITE_P(Http2CodecImplStreamLimitTest, Http2CodecImplStreamLimitTest, - ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, - HTTP2SETTINGS_DEFAULT_COMBINE, - ::testing::Values(Http2Impl::Nghttp2, - Http2Impl::WrappedNghttp2, - Http2Impl::Oghttp2), - ::testing::Bool())); - -INSTANTIATE_TEST_SUITE_P(Http2CodecImplTestDefaultSettings, Http2CodecImplTest, - ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, - HTTP2SETTINGS_DEFAULT_COMBINE, - ::testing::Values(Http2Impl::Nghttp2, - Http2Impl::WrappedNghttp2, - Http2Impl::Oghttp2), - ::testing::Bool())); +INSTANTIATE_TEST_SUITE_P( + Http2CodecImplStreamLimitTest, Http2CodecImplStreamLimitTest, + ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, HTTP2SETTINGS_DEFAULT_COMBINE, + ::testing::Values(Http2Impl::Nghttp2, Http2Impl::Oghttp2), + ::testing::Bool())); + +INSTANTIATE_TEST_SUITE_P( + Http2CodecImplTestDefaultSettings, Http2CodecImplTest, + ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, HTTP2SETTINGS_DEFAULT_COMBINE, + ::testing::Values(Http2Impl::Nghttp2, Http2Impl::Oghttp2), + ::testing::Bool())); #define HTTP2SETTINGS_EDGE_COMBINE \ ::testing::Combine( \ @@ -2564,17 +2502,14 @@ INSTANTIATE_TEST_SUITE_P(Http2CodecImplTestDefaultSettings, Http2CodecImplTest, // Use with caution as any test using this runs 255 times. using Http2CodecImplTestAll = Http2CodecImplTest; -INSTANTIATE_TEST_SUITE_P(Http2CodecImplTestDefaultSettings, Http2CodecImplTestAll, - ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, - HTTP2SETTINGS_DEFAULT_COMBINE, - ::testing::Values(Http2Impl::Nghttp2, - Http2Impl::WrappedNghttp2, - Http2Impl::Oghttp2), - ::testing::Bool())); +INSTANTIATE_TEST_SUITE_P( + Http2CodecImplTestDefaultSettings, Http2CodecImplTestAll, + ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, HTTP2SETTINGS_DEFAULT_COMBINE, + ::testing::Values(Http2Impl::Nghttp2, Http2Impl::Oghttp2), + ::testing::Bool())); INSTANTIATE_TEST_SUITE_P(Http2CodecImplTestEdgeSettings, Http2CodecImplTestAll, ::testing::Combine(HTTP2SETTINGS_EDGE_COMBINE, HTTP2SETTINGS_EDGE_COMBINE, ::testing::Values(Http2Impl::Nghttp2, - Http2Impl::WrappedNghttp2, Http2Impl::Oghttp2), ::testing::Bool())); @@ -2683,13 +2618,11 @@ class Http2CustomSettingsTest ::testing::get<2>(GetParam()), ::testing::get<3>(GetParam()), ::testing::get<4>(GetParam())) {} }; -INSTANTIATE_TEST_SUITE_P(Http2CodecImplTestEdgeSettings, Http2CustomSettingsTest, - ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, - HTTP2SETTINGS_DEFAULT_COMBINE, - ::testing::Values(Http2Impl::Nghttp2, - Http2Impl::WrappedNghttp2, - Http2Impl::Oghttp2), - ::testing::Bool(), ::testing::Bool())); +INSTANTIATE_TEST_SUITE_P( + Http2CodecImplTestEdgeSettings, Http2CustomSettingsTest, + ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, HTTP2SETTINGS_DEFAULT_COMBINE, + ::testing::Values(Http2Impl::Nghttp2, Http2Impl::Oghttp2), ::testing::Bool(), + ::testing::Bool())); // Validates that custom parameters (those which are not explicitly named in the // envoy::config::core::v3::Http2ProtocolOptions proto) are properly sent and processed by @@ -4444,32 +4377,17 @@ class MetadataTestClientConnectionImpl : public TestClientConnectionImpl { // Overrides TestClientConnectionImpl::submitMetadata(). bool submitMetadata(const MetadataMapVector& metadata_map_vector, int32_t stream_id) override { // Creates metadata payload. - if (use_new_codec_wrapper_) { - auto sources = encoder_.createSources(metadata_map_vector); - for (auto& source : sources) { - adapter()->SubmitMetadata(stream_id, 16 * 1024, std::move(source)); - } - int result = adapter()->Send(); - return result == 0; - } else { - encoder_old_.createPayload(metadata_map_vector); - for (uint8_t flags : encoder_old_.payloadFrameFlagBytes()) { - int result = nghttp2_submit_extension(session(), ::Envoy::Http::METADATA_FRAME_TYPE, flags, - stream_id, nullptr); - if (result != 0) { - return false; - } - } - // Triggers nghttp2 to populate the payloads of the METADATA frames. - int result = nghttp2_session_send(session()); - return result == 0; + auto sources = encoder_.createSources(metadata_map_vector); + for (auto& source : sources) { + adapter()->SubmitMetadata(stream_id, 16 * 1024, std::move(source)); } + int result = adapter()->Send(); + return result == 0; } protected: friend class TestNghttp2SessionFactory; - MetadataEncoder encoder_old_; NewMetadataEncoder encoder_; }; @@ -4480,37 +4398,6 @@ class TestNghttp2SessionFactory : public Http2SessionFactory { nghttp2_option_del(options_); } - nghttp2_session* createOld(const nghttp2_session_callbacks*, ConnectionImpl* connection, - const nghttp2_option*) override { - // Only need to provide callbacks required to send METADATA frames. - nghttp2_session_callbacks_new(&callbacks_); - nghttp2_session_callbacks_set_pack_extension_callback( - callbacks_, - [](nghttp2_session*, uint8_t* data, size_t length, const nghttp2_frame*, - void* user_data) -> ssize_t { - // Double cast required due to multiple inheritance. - return static_cast( - static_cast(user_data)) - ->encoder_old_.packNextFramePayload(data, length); - }); - nghttp2_session_callbacks_set_send_callback( - callbacks_, - [](nghttp2_session*, const uint8_t* data, size_t length, int, void* user_data) -> ssize_t { - // Cast down to MetadataTestClientConnectionImpl to leverage friendship. - return static_cast( - static_cast(user_data)) - ->onSend(data, length); - }); - nghttp2_option_new(&options_); - nghttp2_option_set_user_recv_extension_type(options_, METADATA_FRAME_TYPE); - nghttp2_session* session; - nghttp2_session_client_new2(&session, callbacks_, connection, options_); - return session; - } - - void initOld(nghttp2_session*, ConnectionImpl*, - const envoy::config::core::v3::Http2ProtocolOptions&) override {} - std::unique_ptr create(const nghttp2_session_callbacks*, ConnectionImpl* connection, const http2::adapter::OgHttp2Adapter::Options& options) override { diff --git a/test/common/http/http2/codec_impl_test_util.h b/test/common/http/http2/codec_impl_test_util.h index f1ccbc7b7808..59d1b92c8bd4 100644 --- a/test/common/http/http2/codec_impl_test_util.h +++ b/test/common/http/http2/codec_impl_test_util.h @@ -77,14 +77,7 @@ class TestServerConnectionImpl : public TestCodecStatsProvider, max_request_headers_kb, max_request_headers_count, headers_with_underscores_action) {} - nghttp2_session* session() { - ASSERT(!use_new_codec_wrapper_); - return session_; - } - http2::adapter::Http2Adapter* adapter() { - ASSERT(use_new_codec_wrapper_); - return adapter_.get(); - } + http2::adapter::Http2Adapter* adapter() { return adapter_.get(); } using ServerConnectionImpl::getStream; using ServerConnectionImpl::sendPendingFrames; @@ -110,14 +103,7 @@ class TestClientConnectionImpl : public TestCodecStatsProvider, max_request_headers_kb, max_request_headers_count, http2_session_factory) {} - nghttp2_session* session() { - ASSERT(!use_new_codec_wrapper_); - return session_; - } - http2::adapter::Http2Adapter* adapter() { - ASSERT(use_new_codec_wrapper_); - return adapter_.get(); - } + http2::adapter::Http2Adapter* adapter() { return adapter_.get(); } // Submits an H/2 METADATA frame to the peer. // Returns true on success, false otherwise. virtual bool submitMetadata(const MetadataMapVector& mm_vector, int32_t stream_id) { diff --git a/test/integration/BUILD b/test/integration/BUILD index 1a9851083de2..345ef2dae15c 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1679,13 +1679,6 @@ envoy_cc_fuzz_test( deps = [":h2_fuzz_lib"], ) -envoy_cc_fuzz_test( - name = "h2_wrapped_capture_fuzz_test", - srcs = ["h2_wrapped_capture_fuzz_test.cc"], - corpus = "h2_corpus", - deps = [":h2_fuzz_lib"], -) - envoy_cc_fuzz_test( name = "h2_capture_persistent_fuzz_test", srcs = ["h2_capture_fuzz_test.cc"], diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index af3c9339b001..940f0d359d11 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -333,19 +333,10 @@ class TestHttp2ServerConnectionImpl : public Http::Http2::ServerConnectionImpl { headers_with_underscores_action) {} void updateConcurrentStreams(uint32_t max_streams) { - int rc; - if (use_new_codec_wrapper_) { - absl::InlinedVector settings; - settings.insert(settings.end(), {{http2::adapter::MAX_CONCURRENT_STREAMS, max_streams}}); - adapter_->SubmitSettings(settings); - rc = adapter_->Send(); - } else { - absl::InlinedVector settings; - settings.insert(settings.end(), {{NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, max_streams}}); - rc = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, settings.data(), settings.size()); - ASSERT(rc == 0); - rc = nghttp2_session_send(session_); - } + absl::InlinedVector settings; + settings.push_back({http2::adapter::MAX_CONCURRENT_STREAMS, max_streams}); + adapter_->SubmitSettings(settings); + const int rc = adapter_->Send(); ASSERT(rc == 0); } }; diff --git a/test/integration/h2_wrapped_capture_fuzz_test.cc b/test/integration/h2_wrapped_capture_fuzz_test.cc deleted file mode 100644 index 8a8c98a67107..000000000000 --- a/test/integration/h2_wrapped_capture_fuzz_test.cc +++ /dev/null @@ -1,32 +0,0 @@ -#include "test/integration/h2_fuzz.h" - -namespace Envoy { -void H2FuzzIntegrationTest::initialize() { - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() >= 1, ""); - ConfigHelper::HttpProtocolOptions protocol_options; - protocol_options.mutable_explicit_http_config() - ->mutable_http2_protocol_options() - ->set_allow_metadata(true); - ConfigHelper::setProtocolOptions(*bootstrap.mutable_static_resources()->mutable_clusters(0), - protocol_options); - }); - config_helper_.addConfigModifier( - [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& - hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); - config_helper_.addRuntimeOverride("envoy.reloadable_features.http2_new_codec_wrapper", "true"); - setDownstreamProtocol(Http::CodecType::HTTP2); - setUpstreamProtocol(Http::CodecType::HTTP2); - - HttpIntegrationTest::initialize(); -} - -DEFINE_PROTO_FUZZER(const test::integration::H2CaptureFuzzTestCase& input) { - // Pick an IP version to use for loopback, it doesn't matter which. - FUZZ_ASSERT(!TestEnvironment::getIpVersionsForTest().empty()); - const auto ip_version = TestEnvironment::getIpVersionsForTest()[0]; - PERSISTENT_FUZZ_VAR H2FuzzIntegrationTest h2_fuzz_integration_test(ip_version); - h2_fuzz_integration_test.replay(input, false); -} - -} // namespace Envoy diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 6c9042d48b5a..a06f4ca99fdf 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -378,15 +378,9 @@ void HttpIntegrationTest::initialize() { void HttpIntegrationTest::setupHttp2Overrides(Http2Impl implementation) { switch (implementation) { case Http2Impl::Nghttp2: - config_helper_.addRuntimeOverride("envoy.reloadable_features.http2_new_codec_wrapper", "false"); - config_helper_.addRuntimeOverride("envoy.reloadable_features.http2_use_oghttp2", "false"); - break; - case Http2Impl::WrappedNghttp2: - config_helper_.addRuntimeOverride("envoy.reloadable_features.http2_new_codec_wrapper", "true"); config_helper_.addRuntimeOverride("envoy.reloadable_features.http2_use_oghttp2", "false"); break; case Http2Impl::Oghttp2: - config_helper_.addRuntimeOverride("envoy.reloadable_features.http2_new_codec_wrapper", "true"); config_helper_.addRuntimeOverride("envoy.reloadable_features.http2_use_oghttp2", "true"); break; } diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index 6cf568d19731..58ff76c68aff 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -22,7 +22,6 @@ using ::Envoy::Http::Http2::Http2Frame; enum class Http2Impl { Nghttp2, - WrappedNghttp2, Oghttp2, }; diff --git a/test/integration/http_protocol_integration.cc b/test/integration/http_protocol_integration.cc index 191bf95da2ce..3a930772d39b 100644 --- a/test/integration/http_protocol_integration.cc +++ b/test/integration/http_protocol_integration.cc @@ -8,15 +8,13 @@ std::vector HttpProtocolIntegrationTest::getProtocolTest const std::vector& upstream_protocols) { std::vector ret; - const auto addHttp2TestParametersWithNewCodecWrapperOrDeferredProcessing = + const auto addHttp2TestParametersWithDeferredProcessing = [&ret](Network::Address::IpVersion ip_version, Http::CodecType downstream_protocol, Http::CodecType upstream_protocol) { - for (Http2Impl impl : {Http2Impl::WrappedNghttp2, Http2Impl::Oghttp2}) { - ret.push_back(HttpProtocolTestParams{ip_version, downstream_protocol, upstream_protocol, - impl, false}); - ret.push_back(HttpProtocolTestParams{ip_version, downstream_protocol, upstream_protocol, - impl, true}); - } + ret.push_back(HttpProtocolTestParams{ip_version, downstream_protocol, upstream_protocol, + Http2Impl::Oghttp2, false}); + ret.push_back(HttpProtocolTestParams{ip_version, downstream_protocol, upstream_protocol, + Http2Impl::Oghttp2, true}); ret.push_back(HttpProtocolTestParams{ip_version, downstream_protocol, upstream_protocol, Http2Impl::Nghttp2, true}); }; @@ -29,8 +27,8 @@ std::vector HttpProtocolIntegrationTest::getProtocolTest Http2Impl::Nghttp2, false}); if (downstream_protocol == Http::CodecType::HTTP2 || upstream_protocol == Http::CodecType::HTTP2) { - addHttp2TestParametersWithNewCodecWrapperOrDeferredProcessing( - ip_version, downstream_protocol, upstream_protocol); + addHttp2TestParametersWithDeferredProcessing(ip_version, downstream_protocol, + upstream_protocol); } #else if (downstream_protocol == Http::CodecType::HTTP3 || @@ -41,8 +39,8 @@ std::vector HttpProtocolIntegrationTest::getProtocolTest Http2Impl::Nghttp2, false}); if (downstream_protocol == Http::CodecType::HTTP2 || upstream_protocol == Http::CodecType::HTTP2) { - addHttp2TestParametersWithNewCodecWrapperOrDeferredProcessing( - ip_version, downstream_protocol, upstream_protocol); + addHttp2TestParametersWithDeferredProcessing(ip_version, downstream_protocol, + upstream_protocol); } } #endif @@ -80,8 +78,6 @@ absl::string_view implementationToString(Http2Impl impl) { switch (impl) { case Http2Impl::Nghttp2: return "Nghttp2"; - case Http2Impl::WrappedNghttp2: - return "WrappedNghttp2"; case Http2Impl::Oghttp2: return "Oghttp2"; } diff --git a/test/integration/multiplexed_integration_test.cc b/test/integration/multiplexed_integration_test.cc index c64ee4a1fe85..1139c69179c6 100644 --- a/test/integration/multiplexed_integration_test.cc +++ b/test/integration/multiplexed_integration_test.cc @@ -1737,30 +1737,23 @@ TEST_P(MultiplexedRingHashIntegrationTest, CookieRoutingWithCookieWithTtlSet) { struct FrameIntegrationTestParam { Network::Address::IpVersion ip_version; - bool enable_new_codec_wrapper; }; std::string frameIntegrationTestParamToString(const testing::TestParamInfo& params) { const bool is_ipv4 = params.param.ip_version == Network::Address::IpVersion::v4; - const bool new_codec_wrapper = params.param.enable_new_codec_wrapper; - return absl::StrCat(is_ipv4 ? "IPv4" : "IPv6", new_codec_wrapper ? "WrappedNghttp2" : "Nghttp2"); + return is_ipv4 ? "IPv4" : "IPv6"; } class Http2FrameIntegrationTest : public testing::TestWithParam, public Http2RawFrameIntegrationTest { public: - Http2FrameIntegrationTest() : Http2RawFrameIntegrationTest(GetParam().ip_version) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.http2_new_codec_wrapper", - GetParam().enable_new_codec_wrapper ? "true" : "false"); - } + Http2FrameIntegrationTest() : Http2RawFrameIntegrationTest(GetParam().ip_version) {} static std::vector testParams() { std::vector v; for (auto ip_version : TestEnvironment::getIpVersionsForTest()) { - for (bool enable_new_codec_wrapper : {false, true}) { - v.push_back({ip_version, enable_new_codec_wrapper}); - } + v.push_back({ip_version}); } return v; } diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 6b4322960f42..81bf2ea3aaba 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -12,8 +12,8 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/crypto:0.0" "source/common/event:94.1" # Emulated edge events guards don't report LCOV "source/common/filesystem/posix:95.5" -"source/common/http:96.3" -"source/common/http/http2:96.4" +"source/common/http:96.1" +"source/common/http/http2:94.8" "source/common/io:98.0" "source/common/json:89.8" "source/common/matcher:92.0" From 6338eb0fcfc0805823590ff47d5996875dcb4089 Mon Sep 17 00:00:00 2001 From: danzh Date: Fri, 28 Oct 2022 12:14:48 -0400 Subject: [PATCH 029/112] Quiche roll 20221027194109 (#23721) Update QUICHE from 08e3ff8ef to 9624f2d43 https://github.com/google/quiche/compare/08e3ff8ef..9624f2d43 $ git log 08e3ff8ef..9624f2d43 --date=short --no-merges --format="%ad %al %s" 2022-10-27 bnc Remove QUICHE_EXPORT from connect_udp_datagram_payload.h. 2022-10-26 peiwenhu Use std::move to avoid making copies of payload. 2022-10-26 wub Internal change 2022-10-25 fayang Add google_handshake_message transport parameter. 2022-10-25 martinduke Internal change 2022-10-25 wub Allow QuicClientBase to use a fixed server connection id. Remove similar functionality from quic::test::MockableQuicClient. 2022-10-25 martinduke Internal change 2022-10-25 renjietang Internal change 2022-10-24 martinduke Update LoadBalancerEncoder to draft-15. 2022-10-24 kmg Add PrintTo method to BinaryHttpMessage::Field 2022-10-24 ericorth Implement CONNECT-UDP toy server backend 2022-10-21 haoyuewang Internal change 2022-10-21 vasilvv Rename QUICHE_EXPORT_PRIVATE to QUICHE_EXPORT 2022-10-21 quiche-dev Permit strings, tokens, and byte sequences to be moved out of Items 2022-10-20 dmauro LSC: Replace deprecated functions absl::MakeUnique and absl::make_unique with std::make_unique 2022-10-19 quiche-dev oblivious_http: Overloaded Parsing method to read payload using QuicheDataReader 2022-10-19 martinduke Eliminate unexpected calls to ConnectionIdLength() and MaybeReplaceConnectionId(). Fixes several incorrect edge cases in QuicDispatcher and QuicFramer that do not have current observable behaviors in production 2022-10-19 quiche-dev Consolidate definitions of CreateClientObliviousRequestWithSeedForTesting 2022-10-19 vasilvv Fix open-source QUICHE build. 2022-10-19 quiche-dev Add QUICHE_EXPORT_PRIVATE to Binary HTTP PrintTo declaration. 2022-10-18 quiche-dev Internal change 2022-10-18 quiche-dev oblivious_http: Release ownership of ObliviousHttpRequest::Context to callers and remove shared_ptr 2022-10-18 quiche-dev oblivious_http: Cleanup old code 2022-10-18 quiche-dev oblivious_http:Update comments to indicate that Client & Gateway are thread-safe. Signed-off-by: Dan Zhang --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index d57aada18df2..c23cd97ac256 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1080,12 +1080,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "08e3ff8ef734c98ccd681e6819fe2cf8e9db80a9", - sha256 = "2b0fe3c989dc1c612dad07d74cb7d5be2cf495527248f6bbceaf2dedb52c5d39", + version = "9624f2d439f5531ef92346f18d337270cca4ee23", + sha256 = "6d30f11ee198751851b83c50448e9fe9d62dc55c9f897f40be9b35a6fc49039b", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["dataplane_core"], - release_date = "2022-10-18", + release_date = "2022-10-27", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", From 9a17c848100f4fcc96ca7ceebd552ba7606499c5 Mon Sep 17 00:00:00 2001 From: "Vikas Choudhary (vikasc)" Date: Fri, 28 Oct 2022 22:19:00 +0530 Subject: [PATCH 030/112] Fix oauth2 intergration test flake (#23734) Signed-off-by: Vikas Choudhary choudharyvikas16@gmail.com fixes: #23618 Commit Message: Fix oauth2 intergration test flake Additional Description: tried with --gtest_repeat=500 --gtest_break_on_failure, no failure Risk Level: low Testing: Docs Changes: Release Notes: Platform Specific Features: [Optional Runtime guard:] [Optional Fixes #Issue] [Optional Fixes commit #PR or SHA] [Optional Deprecated:] [Optional API Considerations:] --- .../filters/http/oauth2/oauth_integration_test.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/extensions/filters/http/oauth2/oauth_integration_test.cc b/test/extensions/filters/http/oauth2/oauth_integration_test.cc index b4bf427c612b..5fbc0df6ee71 100644 --- a/test/extensions/filters/http/oauth2/oauth_integration_test.cc +++ b/test/extensions/filters/http/oauth2/oauth_integration_test.cc @@ -401,6 +401,13 @@ TEST_P(OauthIntegrationTest, LoadListenerAfterServerIsInitialized) { test_server_->waitForGaugeEq("listener_manager.total_listeners_warming", 0); doAuthenticationFlow("token_secret", "hmac_secret"); + if (lds_connection_ != nullptr) { + AssertionResult result = lds_connection_->close(); + RELEASE_ASSERT(result, result.message()); + result = lds_connection_->waitForDisconnect(); + RELEASE_ASSERT(result, result.message()); + lds_connection_.reset(); + } } class OauthIntegrationTestWithBasicAuth : public OauthIntegrationTest { void setOauthConfig() override { From 2e3e6c5eb92f3bed78bb972a11069ec4a07e2ca0 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Fri, 28 Oct 2022 13:17:28 -0400 Subject: [PATCH 031/112] Fix uninitialized value in the retry policy mock (#23463) Signed-off-by: Yan Avlasov --- test/mocks/router/mocks.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 254c3331dde6..58f013bd56c7 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -141,7 +141,7 @@ class TestRetryPolicy : public RetryPolicy { std::chrono::milliseconds per_try_idle_timeout_{0}; uint32_t num_retries_{}; uint32_t retry_on_{}; - uint32_t host_selection_max_attempts_; + uint32_t host_selection_max_attempts_{0}; std::vector retriable_status_codes_; std::vector retriable_headers_; std::vector retriable_request_headers_; From cd208a5dbc281dcc27a8155a210037267c08ff6f Mon Sep 17 00:00:00 2001 From: Kuo-Chung Hsu Date: Fri, 28 Oct 2022 10:20:03 -0700 Subject: [PATCH 032/112] Thrift: Payload to metadata filter (#23409) This filter is configured with request_rules that will be matched against requests. A field_selector of a rule represents the head of a linked list, each node of the linked list has a name for logging and an id for matching. The field_selector is tied to a payload field when the linked list corresponds to a downward path which rooted in the top-level of the request message structure. on_present is triggered when corresponding the payload is present. Otherwise, on_missing is triggered. This filter is designed to support payload passthrough. By performing payload to metadata filter can do deserialization once, and pass the metadata to other filters. This means that load balancing decisions, consumed from log and routing could all use payload information with a single parse. Also notably performing the parsing in payload passthrough buffer will mean deserialization once and not re-serializing, which is the most performant outcome. Risk Level: low Testing: unit Docs Changes: multiple rst Fixes #23322 Signed-off-by: kuochunghsu --- api/BUILD | 1 + .../filters/payload_to_metadata/v3/BUILD | 12 + .../v3/payload_to_metadata.proto | 100 ++ api/versioning/BUILD | 1 + changelogs/current.yaml | 4 +- .../_include/payload-to-metadata-filter.yaml | 72 + .../thrift_filters/_include/request.proto | 11 + .../payload_to_metadata_filter.rst | 107 ++ .../thrift_filters/thrift_filters.rst | 1 + source/extensions/extensions_build_config.bzl | 1 + source/extensions/extensions_metadata.yaml | 7 + .../filters/payload_to_metadata/BUILD | 36 + .../filters/payload_to_metadata/config.cc | 35 + .../filters/payload_to_metadata/config.h | 33 + .../payload_to_metadata_filter.cc | 319 +++++ .../payload_to_metadata_filter.h | 191 +++ .../header_to_metadata_filter_test.cc | 2 +- .../filters/payload_to_metadata/BUILD | 37 + .../payload_to_metadata/config_test.cc | 225 +++ .../payload_to_metadata_filter_test.cc | 1226 +++++++++++++++++ 20 files changed, 2419 insertions(+), 2 deletions(-) create mode 100644 api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/BUILD create mode 100644 api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/payload_to_metadata.proto create mode 100644 docs/root/configuration/other_protocols/thrift_filters/_include/payload-to-metadata-filter.yaml create mode 100644 docs/root/configuration/other_protocols/thrift_filters/_include/request.proto create mode 100644 docs/root/configuration/other_protocols/thrift_filters/payload_to_metadata_filter.rst create mode 100644 source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/BUILD create mode 100644 source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config.cc create mode 100644 source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config.h create mode 100644 source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.cc create mode 100644 source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.h create mode 100644 test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/BUILD create mode 100644 test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config_test.cc create mode 100644 test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter_test.cc diff --git a/api/BUILD b/api/BUILD index e28c8f3e2553..29414b11265b 100644 --- a/api/BUILD +++ b/api/BUILD @@ -217,6 +217,7 @@ proto_library( "//envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3:pkg", "//envoy/extensions/filters/network/tcp_proxy/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/filters/header_to_metadata/v3:pkg", + "//envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/filters/ratelimit/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/router/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/v3:pkg", diff --git a/api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/BUILD b/api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/BUILD new file mode 100644 index 000000000000..693f0b92ff34 --- /dev/null +++ b/api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/type/matcher/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/payload_to_metadata.proto b/api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/payload_to_metadata.proto new file mode 100644 index 000000000000..894df76e1df3 --- /dev/null +++ b/api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/payload_to_metadata.proto @@ -0,0 +1,100 @@ +syntax = "proto3"; + +package envoy.extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3; + +import "envoy/type/matcher/v3/regex.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3"; +option java_outer_classname = "PayloadToMetadataProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3;payload_to_metadatav3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Payload-To-Metadata Filter] +// +// The configuration for transforming payloads into metadata. This is useful +// for matching load balancer subsets, logging, etc. +// +// Payload to Metadata :ref:`configuration overview `. +// [#extension: envoy.filters.thrift.payload_to_metadata] + +message PayloadToMetadata { + enum ValueType { + STRING = 0; + NUMBER = 1; + } + + // [#next-free-field: 6] + message KeyValuePair { + // The namespace — if this is empty, the filter's namespace will be used. + string metadata_namespace = 1; + + // The key to use within the namespace. + string key = 2 [(validate.rules).string = {min_len: 1}]; + + oneof value_type { + // The value to pair with the given key. + // + // When used for on_present case, if value is non-empty it'll be used instead + // of the field value. If both are empty, the field value is used as-is. + // + // When used for on_missing case, a non-empty value must be provided. + string value = 3; + + // If present, the header's value will be matched and substituted with this. + // If there is no match or substitution, the field value is used as-is. + // + // This is only used for on_present. + type.matcher.v3.RegexMatchAndSubstitute regex_value_rewrite = 4; + } + + // The value's type — defaults to string. + ValueType type = 5 [(validate.rules).enum = {defined_only: true}]; + } + + // A Rule defines what metadata to apply when a field is present or missing. + // [#next-free-field: 6] + message Rule { + oneof match_specifier { + option (validate.required) = true; + + // If specified, the route must exactly match the request method name. As a special case, + // an empty string matches any request method name. + string method_name = 1; + + // If specified, the route must have the service name as the request method name prefix. + // As a special case, an empty string matches any service name. Only relevant when service + // multiplexing. + string service_name = 2; + } + + // Specifies that a match will be performed on the value of a field. + FieldSelector field_selector = 3 [(validate.rules).message = {required: true}]; + + // If the field is present, apply this metadata KeyValuePair. + KeyValuePair on_present = 4; + + // If the field is missing, apply this metadata KeyValuePair. + // + // The value in the KeyValuePair must be set, since it'll be used in lieu + // of the missing field value. + KeyValuePair on_missing = 5; + } + + message FieldSelector { + // field name to log + string name = 1 [(validate.rules).string = {min_len: 1}]; + + // field id to match + int32 id = 2 [(validate.rules).int32 = {lte: 32767 gte: -32768}]; + + // next node of the field selector + FieldSelector child = 3; + } + + // The list of rules to apply to requests. + repeated Rule request_rules = 1 [(validate.rules).repeated = {min_items: 1}]; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 28dd2b0a66bb..a3935c83ab22 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -156,6 +156,7 @@ proto_library( "//envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3:pkg", "//envoy/extensions/filters/network/tcp_proxy/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/filters/header_to_metadata/v3:pkg", + "//envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/filters/ratelimit/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/router/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/v3:pkg", diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 9e0aaab4f80e..1d024a6b2cc8 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -50,7 +50,9 @@ new_features: - area: upstream change: | added a new field :ref:`socket_options ` to the ExtraSourceAddress, allowing specifying discrete socket options for each source address. - +- area: thrift + change: | + added payload to metadata filter which matches a given payload field's value would be extracted and attached to the request as dynamic metadata. - area: http change: | allowing the dynamic forward proxy cluster to :ref:`allow_coalesced_connections ` for HTTP/2 and HTTP/3 connections. diff --git a/docs/root/configuration/other_protocols/thrift_filters/_include/payload-to-metadata-filter.yaml b/docs/root/configuration/other_protocols/thrift_filters/_include/payload-to-metadata-filter.yaml new file mode 100644 index 000000000000..e680242ce7f8 --- /dev/null +++ b/docs/root/configuration/other_protocols/thrift_filters/_include/payload-to-metadata-filter.yaml @@ -0,0 +1,72 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 9090 + filter_chains: + - filters: + - name: envoy.filters.network.thrift_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy + stat_prefix: ingress_thrift + route_config: + name: local_route + routes: + - match: + method_name: "" + route: + cluster: versioned-cluster + thrift_filters: + - name: envoy.filters.thrift.payload_to_metadata + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata + request_rules: + - method_name: foo + field_selector: + name: info + id: 2 + child: + name: version + id: 1 + on_present: + metadata_namespace: envoy.lb + key: version + on_missing: + metadata_namespace: envoy.lb + key: default + value: 'unknown' + clusters: + - name: versioned-cluster + type: STRICT_DNS + lb_policy: ROUND_ROBIN + lb_subset_config: + fallback_policy: NO_FALLBACK + subset_selectors: + - keys: + - default + - keys: + - version + load_assignment: + cluster_name: versioned-cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19090 + metadata: + filter_metadata: + envoy.lb: + default: "true" + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19091 + metadata: + filter_metadata: + envoy.lb: + version: "1.0" diff --git a/docs/root/configuration/other_protocols/thrift_filters/_include/request.proto b/docs/root/configuration/other_protocols/thrift_filters/_include/request.proto new file mode 100644 index 000000000000..764e9337bb10 --- /dev/null +++ b/docs/root/configuration/other_protocols/thrift_filters/_include/request.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package request; + +message Request { + message Info { + string version = 1; + } + string data = 1; + Info info = 2; +} diff --git a/docs/root/configuration/other_protocols/thrift_filters/payload_to_metadata_filter.rst b/docs/root/configuration/other_protocols/thrift_filters/payload_to_metadata_filter.rst new file mode 100644 index 000000000000..0c27eecd498a --- /dev/null +++ b/docs/root/configuration/other_protocols/thrift_filters/payload_to_metadata_filter.rst @@ -0,0 +1,107 @@ +.. _config_thrift_filters_payload_to_metadata: + +Envoy Payload-To-Metadata Filter +================================ +* This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata``. +* :ref:`v3 API reference ` + +A typical use case for this filter is to dynamically match a specified payload field of requests +with load balancer subsets. For this, a given payload field's value would be extracted and attached +to the request as dynamic metadata which would then be used to match a subset of endpoints. + +We already have :ref:`header-To-metadata filter ` +to achieve the similar goal. However, we have two reasons for introducing new :ref:`payload-To-metadata filter +`: + +1. Transports like framed transport don't support THeaders, which is unable to use :ref:`Header-To-Metadata filter +`. + +2. Directly referring to payload field stops envoy relying on that the downstream service always copies the field +to the THeader correctly and guarantees single truth of source. + +This filter is configured with :ref:`request_rules +` +that will be matched against requests. A +:ref:`field_selector +` +of a :ref:`rule +` +represents the head of a linked list, each node of the linked list has a :ref:`name +` +for logging and an :ref:`id +` +for matching. The :ref:`field_selector +` +is tied to a payload field when the linked list corresponds to a downward path which rooted in the top-level of the +request message structure. :ref:`on_present +` +is triggered when corresponding the payload is present. Otherwise, :ref:`on_missing +` +is triggered. + +Note that if the corresponding payload for a :ref:`rule +` +is present but :ref:`on_present +` +is missing, no metadata is added for this :ref:`rule +`. +. If the corresponding payload for a :ref:`rule +` +is an empty string, neither :ref:`on_present +` +nor :ref:`on_missing +` +is triggered. i.e., no metadata is added for this :ref:`rule +`. + +Currently payload to metadata filter doesn't support container type payload, i.e., list, set and map. + +We limit the size of a single metadata value which is added by this filter to 1024 bytes. + +This filter is designed to support payload passthrough. Performing payload to metadata filter +can do deserialization once, and pass the metadata to other filters. This means that load balancing +decisions, consumed from log and routing could all use payload information with a single parse. +Also notably performing the parsing in payload passthrough buffer will mean deserialization once +and not re-serializing, which is the most performant outcome. + +If any of the filter chain doesn't support payload passthrough, a customized non-passthrough +filter to setup metadata is encouraged from point of performance view. + +Example +------- + +A sample filter configuration to route traffic to endpoints based on the presence or +absence of a version payload could be: + +.. literalinclude:: _include/payload-to-metadata-filter.yaml + :language: yaml + :lines: 20-38 + :lineno-start: 20 + :linenos: + :caption: :download:`payload-to-metadata-filter.yaml <_include/payload-to-metadata-filter.yaml>` + +A corresponding upstream cluster configuration could be: + +.. literalinclude:: _include/header-to-metadata-filter.yaml + :language: yaml + :lines: 39-49 + :lineno-start: 37 + :linenos: + :caption: :download:`header-to-metadata-filter.yaml <_include/header-to-metadata-filter.yaml>` + +The request thrift structure could be: + +.. literalinclude:: _include/request.proto + :language: proto + +This would then allow requests of method name ``foo`` with the ``version`` payload field which is +under ``info`` field set to be matched against endpoints with the corresponding version. Whereas +requests with that payload missing would be matched with the default endpoints. + +The regex matching and substitution is similiar with :ref:`header to metadata filter `. + + +Statistics +---------- + +Currently, this filter generates no statistics. diff --git a/docs/root/configuration/other_protocols/thrift_filters/thrift_filters.rst b/docs/root/configuration/other_protocols/thrift_filters/thrift_filters.rst index 3dd28c652332..48e5fd2ece15 100644 --- a/docs/root/configuration/other_protocols/thrift_filters/thrift_filters.rst +++ b/docs/root/configuration/other_protocols/thrift_filters/thrift_filters.rst @@ -9,5 +9,6 @@ Envoy has the following builtin Thrift filters. :maxdepth: 2 header_to_metadata_filter + payload_to_metadata_filter rate_limit_filter router_filter diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 00bf915a9dfd..f97b97e7a4f6 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -186,6 +186,7 @@ EXTENSIONS = { "envoy.filters.thrift.router": "//source/extensions/filters/network/thrift_proxy/router:config", "envoy.filters.thrift.header_to_metadata": "//source/extensions/filters/network/thrift_proxy/filters/header_to_metadata:config", + "envoy.filters.thrift.payload_to_metadata": "//source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata:config", "envoy.filters.thrift.rate_limit": "//source/extensions/filters/network/thrift_proxy/filters/ratelimit:config", # diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index b82f584f6b79..ed3aecce4bae 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -621,6 +621,13 @@ envoy.filters.thrift.header_to_metadata: status: alpha type_urls: - envoy.extensions.filters.network.thrift_proxy.filters.header_to_metadata.v3.HeaderToMetadata +envoy.filters.thrift.payload_to_metadata: + categories: + - envoy.thrift_proxy.filters + security_posture: requires_trusted_downstream_and_upstream + status: alpha + type_urls: + - envoy.extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata envoy.filters.thrift.rate_limit: categories: - envoy.thrift_proxy.filters diff --git a/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/BUILD b/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/BUILD new file mode 100644 index 000000000000..cc6eeb8630a0 --- /dev/null +++ b/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/BUILD @@ -0,0 +1,36 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":payload_to_metadata_filter_lib", + "//envoy/registry", + "//source/extensions/filters/network/thrift_proxy/filters:factory_base_lib", + "@envoy_api//envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "payload_to_metadata_filter_lib", + srcs = ["payload_to_metadata_filter.cc"], + hdrs = ["payload_to_metadata_filter.h"], + deps = [ + "//envoy/server:filter_config_interface", + "//source/extensions/filters/network/thrift_proxy:auto_protocol_lib", + "//source/extensions/filters/network/thrift_proxy:auto_transport_lib", + "//source/extensions/filters/network/thrift_proxy:decoder_lib", + "//source/extensions/filters/network/thrift_proxy/filters:pass_through_filter_lib", + "@envoy_api//envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config.cc b/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config.cc new file mode 100644 index 000000000000..cf211bef9e5b --- /dev/null +++ b/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config.cc @@ -0,0 +1,35 @@ +#include "source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config.h" + +#include + +#include "source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.h" + +namespace Envoy { +namespace Extensions { +namespace ThriftFilters { +namespace PayloadToMetadataFilter { + +using namespace Envoy::Extensions::NetworkFilters; + +ThriftProxy::ThriftFilters::FilterFactoryCb +PayloadToMetadataFilterConfig::createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::network::thrift_proxy::filters::payload_to_metadata::v3:: + PayloadToMetadata& proto_config, + const std::string&, Server::Configuration::FactoryContext&) { + ConfigSharedPtr filter_config(std::make_shared(proto_config)); + return + [filter_config](ThriftProxy::ThriftFilters::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addDecoderFilter(std::make_shared(filter_config)); + }; +} + +/** + * Static registration for the header to metadata filter. @see RegisterFactory. + */ +REGISTER_FACTORY(PayloadToMetadataFilterConfig, + ThriftProxy::ThriftFilters::NamedThriftFilterConfigFactory); + +} // namespace PayloadToMetadataFilter +} // namespace ThriftFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config.h b/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config.h new file mode 100644 index 000000000000..520aeb14b14a --- /dev/null +++ b/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config.h @@ -0,0 +1,33 @@ +#pragma once + +#include "envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/payload_to_metadata.pb.h" +#include "envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/payload_to_metadata.pb.validate.h" + +#include "source/extensions/filters/network/thrift_proxy/filters/factory_base.h" +#include "source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.h" + +namespace Envoy { +namespace Extensions { +namespace ThriftFilters { +namespace PayloadToMetadataFilter { + +/** + * Config registration for the header to metadata filter. @see NamedThriftFilterConfigFactory. + */ +class PayloadToMetadataFilterConfig : public ThriftProxy::ThriftFilters::FactoryBase< + envoy::extensions::filters::network::thrift_proxy:: + filters::payload_to_metadata::v3::PayloadToMetadata> { +public: + PayloadToMetadataFilterConfig() : FactoryBase("envoy.filters.thrift.payload_to_metadata") {} + +private: + ThriftProxy::ThriftFilters::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::network::thrift_proxy::filters::payload_to_metadata::v3:: + PayloadToMetadata& proto_config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; +}; + +} // namespace PayloadToMetadataFilter +} // namespace ThriftFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.cc b/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.cc new file mode 100644 index 000000000000..5476eb5720ff --- /dev/null +++ b/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.cc @@ -0,0 +1,319 @@ +#include "source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.h" + +#include "source/common/common/regex.h" +#include "source/common/network/utility.h" +#include "source/extensions/filters/network/thrift_proxy/auto_protocol_impl.h" +#include "source/extensions/filters/network/thrift_proxy/auto_transport_impl.h" +#include "source/extensions/filters/network/thrift_proxy/binary_protocol_impl.h" +#include "source/extensions/filters/network/thrift_proxy/compact_protocol_impl.h" +#include "source/extensions/filters/network/thrift_proxy/framed_transport_impl.h" +#include "source/extensions/filters/network/thrift_proxy/header_transport_impl.h" +#include "source/extensions/filters/network/thrift_proxy/unframed_transport_impl.h" + +namespace Envoy { +namespace Extensions { +namespace ThriftFilters { +namespace PayloadToMetadataFilter { + +using namespace Envoy::Extensions::NetworkFilters; +using FieldSelector = envoy::extensions::filters::network::thrift_proxy::filters:: + payload_to_metadata::v3::PayloadToMetadata::FieldSelector; + +Config::Config(const envoy::extensions::filters::network::thrift_proxy::filters:: + payload_to_metadata::v3::PayloadToMetadata& config) { + trie_root_ = std::make_shared(); + request_rules_.reserve(config.request_rules().size()); + for (const auto& entry : config.request_rules()) { + request_rules_.emplace_back(entry, static_cast(request_rules_.size()), trie_root_); + } +} + +Rule::Rule(const ProtoRule& rule, uint16_t rule_id, TrieSharedPtr root) + : rule_(rule), rule_id_(rule_id) { + if (!rule_.has_on_present() && !rule_.has_on_missing()) { + throw EnvoyException("payload to metadata filter: neither `on_present` nor `on_missing` set"); + } + + if (rule_.has_on_missing() && rule_.on_missing().value().empty()) { + throw EnvoyException( + "payload to metadata filter: cannot specify on_missing rule without non-empty value"); + } + + if (rule_.has_on_present() && rule_.on_present().has_regex_value_rewrite()) { + const auto& rewrite_spec = rule_.on_present().regex_value_rewrite(); + regex_rewrite_ = Regex::Utility::parseRegex(rewrite_spec.pattern()); + regex_rewrite_substitution_ = rewrite_spec.substitution(); + } + + switch (rule_.match_specifier_case()) { + case ProtoRule::MatchSpecifierCase::kMethodName: + match_type_ = MatchType::MethodName; + method_or_service_name_ = rule_.method_name(); + break; + case ProtoRule::MatchSpecifierCase::kServiceName: + match_type_ = MatchType::ServiceName; + if (!rule_.service_name().empty() && !absl::EndsWith(rule_.service_name(), ":")) { + method_or_service_name_ = rule_.service_name() + ":"; + } else { + method_or_service_name_ = rule_.service_name(); + } + break; + case ProtoRule::MatchSpecifierCase::MATCH_SPECIFIER_NOT_SET: + PANIC_DUE_TO_CORRUPT_ENUM; + } + + const FieldSelector* field_selector = &rule_.field_selector(); + TrieSharedPtr node = root; + while (true) { + int16_t id = static_cast(field_selector->id()); + if (node->children_.find(id) == node->children_.end()) { + node->children_[id] = std::make_shared(node); + } + node = node->children_[id]; + node->name_ = field_selector->name(); + if (!field_selector->has_child()) { + break; + } + + field_selector = &field_selector->child(); + } + + node->rule_ids_.push_back(rule_id_); +} + +bool Rule::matches(const ThriftProxy::MessageMetadata& metadata) const { + if (match_type_ == MatchType::MethodName) { + const std::string& method_name{method_or_service_name_}; + return method_name.empty() || + (metadata.hasMethodName() && metadata.methodName() == method_name); + } + ASSERT(match_type_ == MatchType::ServiceName); + const std::string& service_name{method_or_service_name_}; + return service_name.empty() || + (metadata.hasMethodName() && absl::StartsWith(metadata.methodName(), service_name)); +} + +FilterStatus TrieMatchHandler::messageEnd() { + ENVOY_LOG(trace, "TrieMatchHandler messageEnd"); + parent_.handleOnMissing(); + complete_ = true; + return FilterStatus::Continue; +} + +FilterStatus TrieMatchHandler::structBegin(absl::string_view) { + ENVOY_LOG(trace, "TrieMatchHandler structBegin id:{}", + last_field_id_.has_value() ? std::to_string(last_field_id_.value()) + : "top_level_struct"); + ASSERT(node_); + if (last_field_id_.has_value()) { + if (steps_ == 0 && node_->children_.find(last_field_id_.value()) != node_->children_.end()) { + node_ = node_->children_[last_field_id_.value()]; + ENVOY_LOG(trace, "name: {}", node_->name_); + } else { + steps_++; + } + } + return FilterStatus::Continue; +} + +FilterStatus TrieMatchHandler::structEnd() { + ENVOY_LOG(trace, "TrieMatchHandler structEnd"); + ASSERT(node_); + if (steps_ > 0) { + steps_--; + } else if (node_->parent_.lock()) { + node_ = node_->parent_.lock(); + } else { + // last decoder event + node_ = nullptr; + } + return FilterStatus::Continue; +} + +FilterStatus TrieMatchHandler::fieldBegin(absl::string_view, FieldType&, int16_t& field_id) { + last_field_id_ = field_id; + return FilterStatus::Continue; +} + +FilterStatus TrieMatchHandler::fieldEnd() { + last_field_id_.reset(); + return FilterStatus::Continue; +} + +FilterStatus TrieMatchHandler::stringValue(absl::string_view value) { + ASSERT(last_field_id_.has_value()); + ENVOY_LOG(trace, "TrieMatchHandler stringValue id:{} value:{}", last_field_id_.value(), value); + return handleString(static_cast(value)); +} + +template FilterStatus TrieMatchHandler::numberValue(NumberType value) { + ASSERT(last_field_id_.has_value()); + ENVOY_LOG(trace, "TrieMatchHandler numberValue id:{} value:{}", last_field_id_.value(), value); + return handleString(std::to_string(value)); +} + +FilterStatus TrieMatchHandler::handleString(std::string value) { + ASSERT(node_); + ASSERT(last_field_id_.has_value()); + if (steps_ == 0 && node_->children_.find(last_field_id_.value()) != node_->children_.end() && + !node_->children_[last_field_id_.value()]->rule_ids_.empty()) { + auto on_present_node = node_->children_[last_field_id_.value()]; + ENVOY_LOG(trace, "name: {}", on_present_node->name_); + parent_.handleOnPresent(std::move(value), on_present_node->rule_ids_); + } + return FilterStatus::Continue; +} + +PayloadToMetadataFilter::PayloadToMetadataFilter(const ConfigSharedPtr config) : config_(config) {} + +void PayloadToMetadataFilter::handleOnPresent(std::string&& value, + const std::vector& rule_ids) { + for (uint16_t rule_id : rule_ids) { + if (matched_rule_ids_.find(rule_id) == matched_rule_ids_.end()) { + return; + } + ENVOY_LOG(trace, "handleOnPresent rule_id {}", rule_id); + + matched_rule_ids_.erase(rule_id); + ASSERT(rule_id < config_->requestRules().size()); + const Rule& rule = config_->requestRules()[rule_id]; + if (!value.empty() && rule.rule().has_on_present()) { + // We can *not* always std::move(value) here since we need `value` if multiple rules are + // matched. Optimize the most common usage, which is one rule per payload field. + if (rule_ids.size() == 1) { + applyKeyValue(std::move(value), rule, rule.rule().on_present()); + break; + } else { + applyKeyValue(value, rule, rule.rule().on_present()); + } + } + } +} + +void PayloadToMetadataFilter::handleOnMissing() { + ENVOY_LOG(trace, "{} rules missing", matched_rule_ids_.size()); + + for (uint16_t rule_id : matched_rule_ids_) { + ENVOY_LOG(trace, "handling on_missing rule_id {}", rule_id); + + ASSERT(rule_id < config_->requestRules().size()); + const Rule& rule = config_->requestRules()[rule_id]; + if (!rule.rule().has_on_missing()) { + continue; + } + applyKeyValue("", rule, rule.rule().on_missing()); + } +} + +const std::string& PayloadToMetadataFilter::decideNamespace(const std::string& nspace) const { + static const std::string& payloadToMetadata = "envoy.filters.thrift.payload_to_metadata"; + return nspace.empty() ? payloadToMetadata : nspace; +} + +bool PayloadToMetadataFilter::addMetadata(const std::string& meta_namespace, const std::string& key, + std::string value, ValueType type) { + ProtobufWkt::Value val; + ASSERT(!value.empty()); + + if (value.size() >= MAX_PAYLOAD_VALUE_LEN) { + // Too long, go away. + ENVOY_LOG(error, "metadata value is too long"); + return false; + } + + ENVOY_LOG(trace, "add metadata ns:{} key:{} value:{}", meta_namespace, key, value); + + // Sane enough, add the key/value. + switch (type) { + PANIC_ON_PROTO_ENUM_SENTINEL_VALUES; + case envoy::extensions::filters::network::thrift_proxy::filters::payload_to_metadata::v3:: + PayloadToMetadata::STRING: + val.set_string_value(std::move(value)); + break; + case envoy::extensions::filters::network::thrift_proxy::filters::payload_to_metadata::v3:: + PayloadToMetadata::NUMBER: { + double dval; + if (absl::SimpleAtod(StringUtil::trim(value), &dval)) { + val.set_number_value(dval); + } else { + ENVOY_LOG(debug, "value to number conversion failed"); + return false; + } + break; + } + } + + // Have we seen this namespace before? + auto namespace_iter = structs_by_namespace_.find(meta_namespace); + if (namespace_iter == structs_by_namespace_.end()) { + structs_by_namespace_[meta_namespace] = ProtobufWkt::Struct(); + namespace_iter = structs_by_namespace_.find(meta_namespace); + } + + auto& keyval = namespace_iter->second; + (*keyval.mutable_fields())[key] = val; + + return true; +} + +// add metadata['key']= value depending on payload present or missing case +void PayloadToMetadataFilter::applyKeyValue(std::string value, const Rule& rule, + const KeyValuePair& keyval) { + if (keyval.has_regex_value_rewrite()) { + const auto& matcher = rule.regexRewrite(); + value = matcher->replaceAll(value, rule.regexSubstitution()); + } else if (!keyval.value().empty()) { + value = keyval.value(); + } + // We do *not* modify value is not present in `on_present` or `on_missing`. + // That is, we apply the original field value to the metadata. + + if (!value.empty()) { + const auto& nspace = decideNamespace(keyval.metadata_namespace()); + addMetadata(nspace, keyval.key(), std::move(value), keyval.type()); + } else { + ENVOY_LOG(debug, "value is empty, not adding metadata"); + } +} + +void PayloadToMetadataFilter::finalizeDynamicMetadata() { + if (!structs_by_namespace_.empty()) { + for (auto const& entry : structs_by_namespace_) { + decoder_callbacks_->streamInfo().setDynamicMetadata(entry.first, entry.second); + } + } +} + +FilterStatus PayloadToMetadataFilter::messageBegin(MessageMetadataSharedPtr metadata) { + for (const auto& rule : config_->requestRules()) { + if (rule.matches(*metadata)) { + matched_rule_ids_.insert(rule.ruleId()); + } + } + + ENVOY_LOG(trace, "{} rules matched", matched_rule_ids_.size()); + return FilterStatus::Continue; +} + +FilterStatus PayloadToMetadataFilter::passthroughData(Buffer::Instance& data) { + if (!matched_rule_ids_.empty()) { + ASSERT(!decoder_); + handler_ = std::make_unique(*this, config_->trieRoot()); + transport_ = createTransport(decoder_callbacks_->downstreamTransportType()); + protocol_ = createProtocol(decoder_callbacks_->downstreamProtocolType()); + decoder_ = std::make_unique(*transport_, *protocol_, *handler_); + + bool underflow = false; + decoder_->onData(data, underflow); + ASSERT(handler_->isComplete() || underflow); + + finalizeDynamicMetadata(); + } + + return FilterStatus::Continue; +} + +} // namespace PayloadToMetadataFilter +} // namespace ThriftFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.h b/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.h new file mode 100644 index 000000000000..a273c78bd019 --- /dev/null +++ b/source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.h @@ -0,0 +1,191 @@ +#pragma once + +#include +#include +#include + +#include "envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/payload_to_metadata.pb.h" + +#include "source/common/common/logger.h" +#include "source/common/common/matchers.h" +#include "source/extensions/filters/network/thrift_proxy/decoder.h" +#include "source/extensions/filters/network/thrift_proxy/filters/pass_through_filter.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Extensions { +namespace ThriftFilters { +namespace PayloadToMetadataFilter { + +using namespace Envoy::Extensions::NetworkFilters; +using namespace Envoy::Extensions::NetworkFilters::ThriftProxy; +using ProtoRule = envoy::extensions::filters::network::thrift_proxy::filters::payload_to_metadata:: + v3::PayloadToMetadata::Rule; +using KeyValuePair = envoy::extensions::filters::network::thrift_proxy::filters:: + payload_to_metadata::v3::PayloadToMetadata::KeyValuePair; +using ValueType = envoy::extensions::filters::network::thrift_proxy::filters::payload_to_metadata:: + v3::PayloadToMetadata::ValueType; + +struct Trie; +using TrieSharedPtr = std::shared_ptr; + +class Rule { +public: + Rule(const ProtoRule& rule, uint16_t rule_id, TrieSharedPtr root); + const ProtoRule& rule() const { return rule_; } + const Regex::CompiledMatcherPtr& regexRewrite() const { return regex_rewrite_; } + const std::string& regexSubstitution() const { return regex_rewrite_substitution_; } + uint16_t ruleId() const { return rule_id_; } + bool matches(const ThriftProxy::MessageMetadata& metadata) const; + +private: + enum class MatchType { + MethodName = 1, + ServiceName = 2, + }; + const ProtoRule rule_; + Regex::CompiledMatcherPtr regex_rewrite_{}; + std::string regex_rewrite_substitution_{}; + std::string method_or_service_name_{}; + MatchType match_type_; + uint16_t rule_id_; +}; + +using PayloadToMetadataRules = std::vector; + +struct Trie { + Trie(TrieSharedPtr parent = nullptr) : parent_(parent) {} + std::string name_; + std::weak_ptr parent_; + // Field ID to payload node + absl::node_hash_map children_; + std::vector rule_ids_; +}; + +class Config { +public: + Config(const envoy::extensions::filters::network::thrift_proxy::filters::payload_to_metadata::v3:: + PayloadToMetadata& config); + const PayloadToMetadataRules& requestRules() const { return request_rules_; } + TrieSharedPtr trieRoot() const { return trie_root_; }; + +private: + PayloadToMetadataRules request_rules_; + TrieSharedPtr trie_root_; +}; + +using ConfigSharedPtr = std::shared_ptr; + +class MetadataHandler { +public: + virtual ~MetadataHandler() = default; + virtual void handleOnPresent(std::string&& value, const std::vector& rule_ids) PURE; + virtual void handleOnMissing() PURE; +}; + +class TrieMatchHandler : public DecoderCallbacks, + public PassThroughDecoderEventHandler, + protected Logger::Loggable { +public: + TrieMatchHandler(MetadataHandler& parent, TrieSharedPtr root) : parent_(parent), node_(root) {} + + // DecoderEventHandler + FilterStatus messageEnd() override; + FilterStatus structBegin(absl::string_view) override; + FilterStatus structEnd() override; + FilterStatus fieldBegin(absl::string_view name, FieldType& field_type, + int16_t& field_id) override; + FilterStatus fieldEnd() override; + FilterStatus boolValue(bool& value) override { return numberValue(value); } + FilterStatus byteValue(uint8_t& value) override { return numberValue(value); } + FilterStatus int16Value(int16_t& value) override { return numberValue(value); } + FilterStatus int32Value(int32_t& value) override { return numberValue(value); } + FilterStatus int64Value(int64_t& value) override { return numberValue(value); } + FilterStatus doubleValue(double& value) override { return numberValue(value); } + FilterStatus stringValue(absl::string_view value) override; + FilterStatus mapBegin(FieldType&, FieldType&, uint32_t&) override { + return handleContainerBegin(); + } + FilterStatus mapEnd() override { return handleContainerEnd(); } + FilterStatus listBegin(FieldType&, uint32_t&) override { return handleContainerBegin(); } + FilterStatus listEnd() override { return handleContainerEnd(); } + FilterStatus setBegin(FieldType&, uint32_t&) override { return handleContainerBegin(); } + FilterStatus setEnd() override { return handleContainerEnd(); } + + // DecoderCallbacks + DecoderEventHandler& newDecoderEventHandler() override { return *this; } + bool passthroughEnabled() const override { return false; } + bool isRequest() const override { return true; } + bool headerKeysPreserveCase() const override { return false; } + + bool isComplete() const { return complete_; }; + +private: + template FilterStatus numberValue(NumberType value); + FilterStatus handleString(std::string value); + + FilterStatus handleContainerBegin() { + steps_++; + return FilterStatus::Continue; + } + + FilterStatus handleContainerEnd() { + steps_--; + return FilterStatus::Continue; + } + + MetadataHandler& parent_; + TrieSharedPtr node_; + bool complete_{false}; + absl::optional last_field_id_; + uint16_t steps_{0}; +}; + +using TrieMatchHandlerPtr = std::unique_ptr; + +const uint32_t MAX_PAYLOAD_VALUE_LEN = 8 * 1024; + +class PayloadToMetadataFilter : public MetadataHandler, + public ThriftProxy::ThriftFilters::PassThroughDecoderFilter, + protected Logger::Loggable { +public: + PayloadToMetadataFilter(const ConfigSharedPtr config); + + // DecoderFilter + FilterStatus messageBegin(MessageMetadataSharedPtr metadata) override; + FilterStatus passthroughData(Buffer::Instance& data) override; + + // MetadataHandler + void handleOnPresent(std::string&& value, const std::vector& rule_ids) override; + void handleOnMissing() override; + +private: + static TransportPtr createTransport(TransportType transport) { + return NamedTransportConfigFactory::getFactory(transport).createTransport(); + } + + static ProtocolPtr createProtocol(ProtocolType protocol) { + return NamedProtocolConfigFactory::getFactory(protocol).createProtocol(); + } + + // TODO(kuochunghsu): extract the metadata handling logic form header/payload to metadata filters. + using StructMap = std::map; + bool addMetadata(const std::string&, const std::string&, std::string, ValueType); + void applyKeyValue(std::string, const Rule&, const KeyValuePair&); + const std::string& decideNamespace(const std::string& nspace) const; + void finalizeDynamicMetadata(); + + const ConfigSharedPtr config_; + absl::flat_hash_set matched_rule_ids_; + TrieMatchHandlerPtr handler_; + TransportPtr transport_; + ProtocolPtr protocol_; + DecoderPtr decoder_; + StructMap structs_by_namespace_; +}; + +} // namespace PayloadToMetadataFilter +} // namespace ThriftFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/thrift_proxy/filters/header_to_metadata/header_to_metadata_filter_test.cc b/test/extensions/filters/network/thrift_proxy/filters/header_to_metadata/header_to_metadata_filter_test.cc index d3e6b85cf84a..33f13f377773 100644 --- a/test/extensions/filters/network/thrift_proxy/filters/header_to_metadata/header_to_metadata_filter_test.cc +++ b/test/extensions/filters/network/thrift_proxy/filters/header_to_metadata/header_to_metadata_filter_test.cc @@ -198,7 +198,7 @@ TEST_F(HeaderToMetadataTest, SubstituteEmptyValueTest) { } /** - * Test the value gets written as a number. + * Test the invalid value won't get written as a number. */ TEST_F(HeaderToMetadataTest, NumberTypeTest) { const std::string request_config_yaml = R"EOF( diff --git a/test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/BUILD b/test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/BUILD new file mode 100644 index 000000000000..2c45adcc5ae8 --- /dev/null +++ b/test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/BUILD @@ -0,0 +1,37 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.filters.thrift.payload_to_metadata"], + deps = [ + "//source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata:config", + "//source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata:payload_to_metadata_filter_lib", + "//test/extensions/filters/network/thrift_proxy:mocks", + "//test/mocks/server:server_mocks", + "@envoy_api//envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "payload_to_metadata_filter_test", + srcs = ["payload_to_metadata_filter_test.cc"], + extension_names = ["envoy.filters.thrift.payload_to_metadata"], + deps = [ + "//source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata:payload_to_metadata_filter_lib", + "//test/extensions/filters/network/thrift_proxy:mocks", + "//test/mocks/server:server_mocks", + "//test/mocks/ssl:ssl_mocks", + ], +) diff --git a/test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config_test.cc b/test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config_test.cc new file mode 100644 index 000000000000..8dacdf50e294 --- /dev/null +++ b/test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config_test.cc @@ -0,0 +1,225 @@ +#include + +#include "envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/payload_to_metadata.pb.h" +#include "envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/payload_to_metadata.pb.validate.h" + +#include "source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/config.h" +#include "source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.h" + +#include "test/extensions/filters/network/thrift_proxy/mocks.h" +#include "test/mocks/server/factory_context.h" +#include "test/mocks/server/instance.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace ThriftFilters { +namespace PayloadToMetadataFilter { + +using PayloadToMetadataProtoConfig = envoy::extensions::filters::network::thrift_proxy::filters:: + payload_to_metadata::v3::PayloadToMetadata; + +void testValidConfig(const std::string& yaml) { + PayloadToMetadataProtoConfig proto_config; + TestUtility::loadFromYamlAndValidate(yaml, proto_config); + testing::NiceMock context; + PayloadToMetadataFilterConfig factory; + auto cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + NetworkFilters::ThriftProxy::ThriftFilters::MockFilterChainFactoryCallbacks filter_callbacks; + EXPECT_CALL(filter_callbacks, addDecoderFilter(_)); + cb(filter_callbacks); +} + +// Tests that a valid config with payload is properly consumed. +TEST(PayloadToMetadataFilterConfigTest, SimpleConfig) { + const std::string yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: info + id: 2 + child: + name: version + id: 1 + on_present: + metadata_namespace: envoy.lb + key: version + on_missing: + metadata_namespace: envoy.lb + key: default + value: unknown + )EOF"; + + testValidConfig(yaml); +} + +TEST(PayloadToMetadataFilterConfigTest, SimpleConfigWithServiceName) { + const std::string yaml = R"EOF( +request_rules: + - service_name: foo + field_selector: + name: info + id: 2 + child: + name: version + id: 1 + child: + name: bar + id: 100 + on_present: + metadata_namespace: envoy.lb + key: version + on_missing: + metadata_namespace: envoy.lb + key: default + value: unknown + )EOF"; + + testValidConfig(yaml); +} + +TEST(PayloadToMetadataFilterConfigTest, SimpleConfigWithServiceNameEndInColon) { + const std::string yaml = R"EOF( +request_rules: + - service_name: "foo:" + field_selector: + name: info + id: 2 + on_present: + metadata_namespace: envoy.lb + key: version + on_missing: + metadata_namespace: envoy.lb + key: default + value: unknown + )EOF"; + + testValidConfig(yaml); +} + +// does not allow on_missing without value +TEST(PayloadToMetadataFilterConfigTest, MultipleRulesWithSameFieldSelector) { + std::string yaml = + R"EOF( +request_rules: + - method_name: foo + field_selector: + name: info + id: 2 + on_missing: + metadata_namespace: envoy.lb + key: foo + value: unknown + - method_name: foo + field_selector: + name: info + id: 2 + on_missing: + metadata_namespace: envoy.lb + key: bar + value: unknown + )EOF"; + + testValidConfig(yaml); +} + +// Tests that configuration does not allow value and regex_value_rewrite in the same rule. +TEST(PayloadToMetadataFilterConfigTest, ValueAndRegex) { + const std::string yaml = R"EOF( +request_rules: +- method_name: foo + field_selector: + name: info + id: 2 + on_present: + metadata_namespace: envoy.lb + key: cluster + value: foo + regex_value_rewrite: + pattern: + google_re2: {} + regex: "^/(cluster[\\d\\w-]+)/?.*$" + substitution: "\\1" + )EOF"; + + PayloadToMetadataProtoConfig proto_config; + EXPECT_THROW(TestUtility::loadFromYamlAndValidate(yaml, proto_config), EnvoyException); +} + +// Both method name and service name are not presented. +TEST(PayloadToMetadataFilterConfigTest, EmptyMethodNameAndServiceName) { + const std::string yaml = R"EOF( +request_rules: + - field_selector: + name: info + id: 2 + child: + name: version + id: 1 + on_present: + metadata_namespace: envoy.lb + key: version + on_missing: + metadata_namespace: envoy.lb + key: default + value: unknown + )EOF"; + + PayloadToMetadataProtoConfig proto_config; + EXPECT_THROW(TestUtility::loadFromYamlAndValidate(yaml, proto_config), ProtoValidationException); +} + +void testForbiddenConfig(const std::string& yaml, const std::string& message) { + PayloadToMetadataProtoConfig proto_config; + TestUtility::loadFromYamlAndValidate(yaml, proto_config); + + testing::NiceMock context; + PayloadToMetadataFilterConfig factory; + + EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, "stats", context), + EnvoyException, message); +} + +// does not allow rule without either on_present or on_missing +TEST(PayloadToMetadataFilterConfigTest, EmptyOnPresentOnMissing) { + std::string yaml = + R"EOF( +request_rules: + - method_name: foo + field_selector: + name: info + id: 2 + )EOF"; + std::string error_message = + "payload to metadata filter: neither `on_present` nor `on_missing` set"; + + testForbiddenConfig(yaml, error_message); +} + +// does not allow on_missing without value +TEST(PayloadToMetadataFilterConfigTest, NoValueInOnMissing) { + std::string yaml = + R"EOF( +request_rules: + - method_name: foo + field_selector: + name: info + id: 2 + on_missing: + metadata_namespace: envoy.lb + key: foo + type: STRING + )EOF"; + std::string error_message = + "payload to metadata filter: cannot specify on_missing rule without non-empty value"; + + testForbiddenConfig(yaml, error_message); +} + +} // namespace PayloadToMetadataFilter +} // namespace ThriftFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter_test.cc b/test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter_test.cc new file mode 100644 index 000000000000..a440d4f8467d --- /dev/null +++ b/test/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter_test.cc @@ -0,0 +1,1226 @@ +#include + +#include "source/common/common/base64.h" +#include "source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/payload_to_metadata_filter.h" + +#include "test/extensions/filters/network/thrift_proxy/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/mocks/server/mocks.h" +#include "test/mocks/stream_info/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace ThriftFilters { +namespace PayloadToMetadataFilter { + +namespace { + +MATCHER_P(MapEq, rhs, "") { + const ProtobufWkt::Struct& obj = arg; + EXPECT_TRUE(!rhs.empty()); + for (auto const& entry : rhs) { + EXPECT_NE(obj.fields().find(entry.first), obj.fields().end()); + EXPECT_EQ(obj.fields().at(entry.first).string_value(), entry.second); + } + return true; +} + +MATCHER_P(MapEqNum, rhs, "") { + const ProtobufWkt::Struct& obj = arg; + EXPECT_TRUE(!rhs.empty()); + for (auto const& entry : rhs) { + EXPECT_NE(obj.fields().find(entry.first), obj.fields().end()); + EXPECT_EQ(obj.fields().at(entry.first).number_value(), entry.second); + } + return true; +} + +} // namespace + +using namespace Envoy::Extensions::NetworkFilters; + +class PayloadToMetadataTest : public testing::Test { +public: + void initializeFilter(const std::string& yaml, bool expect_type_inquiry = true) { + envoy::extensions::filters::network::thrift_proxy::filters::payload_to_metadata::v3:: + PayloadToMetadata proto_config; + TestUtility::loadFromYaml(yaml, proto_config); + const auto& filter_config = std::make_shared(proto_config); + filter_ = std::make_shared(filter_config); + filter_->setDecoderFilterCallbacks(decoder_callbacks_); + if (expect_type_inquiry) { + EXPECT_CALL(decoder_callbacks_, downstreamTransportType()).WillOnce(Return(transport_)); + EXPECT_CALL(decoder_callbacks_, downstreamProtocolType()).WillOnce(Return(protocol_)); + } else { + EXPECT_CALL(decoder_callbacks_, downstreamTransportType()).Times(0); + EXPECT_CALL(decoder_callbacks_, downstreamProtocolType()).Times(0); + } + } + + // Request payload + // { + // first_field: 1, + // second_field: "two" or string_value, + // third_field: { + // f1: true, + // f2: 2, + // ... + // f6: 6, + // f7: "seven", + // f8: map, + // f9: list, + // f10: set + // } + // } + void writeMessage(MessageMetadata& metadata, Buffer::Instance& buffer, + std::string string_value = "two") { + Buffer::OwnedImpl msg; + ProtocolPtr proto = NamedProtocolConfigFactory::getFactory(protocol_).createProtocol(); + metadata.setProtocol(protocol_); + metadata.setMethodName("foo"); + metadata.setMessageType(MessageType::Call); + metadata.setSequenceId(0); + + proto->writeMessageBegin(msg, metadata); + proto->writeStructBegin(msg, "wrapper"); + proto->writeFieldBegin(msg, "first_field", FieldType::I64, 1); + proto->writeInt64(msg, 1); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "second_field", FieldType::String, 2); + proto->writeString(msg, string_value); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "third_field", FieldType::Struct, 3); + + proto->writeStructBegin(msg, "payload"); + proto->writeFieldBegin(msg, "f1", FieldType::Bool, 1); + proto->writeBool(msg, true); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "f2", FieldType::Byte, 2); + proto->writeByte(msg, 2); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "f3", FieldType::Double, 3); + proto->writeDouble(msg, 3.0); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "f4", FieldType::I16, 4); + proto->writeInt16(msg, 4); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "f5", FieldType::I32, 5); + proto->writeInt32(msg, 5); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "f6", FieldType::I64, 6); + proto->writeInt64(msg, 6); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "f7", FieldType::String, 7); + proto->writeString(msg, "seven"); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "f8", FieldType::Map, 8); + proto->writeMapBegin(msg, FieldType::I32, FieldType::I32, 1); + proto->writeInt32(msg, 8); + proto->writeInt32(msg, 8); + proto->writeMapEnd(msg); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "f9", FieldType::List, 9); + proto->writeListBegin(msg, FieldType::I32, 1); + proto->writeInt32(msg, 8); + proto->writeListEnd(msg); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "f10", FieldType::Set, 10); + proto->writeSetBegin(msg, FieldType::I32, 1); + proto->writeInt32(msg, 8); + proto->writeSetEnd(msg); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "", FieldType::Stop, 0); // payload stop field + proto->writeStructEnd(msg); + proto->writeFieldEnd(msg); + + proto->writeFieldBegin(msg, "", FieldType::Stop, 0); // wrapper stop field + proto->writeStructEnd(msg); + proto->writeMessageEnd(msg); + + TransportPtr transport = NamedTransportConfigFactory::getFactory(transport_).createTransport(); + transport->encodeFrame(buffer, metadata, msg); + } + + NiceMock decoder_callbacks_; + NiceMock req_info_; + std::shared_ptr filter_; + const ProtocolType protocol_{ProtocolType::Binary}; + const TransportType transport_{TransportType::Header}; +}; + +TEST_F(PayloadToMetadataTest, MatchFirstLayerString) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "two"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, MatchSecondLayerString) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: payload + id: 3 + child: + name: f7 + id: 7 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "seven"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, MatchFirstLayerNumber) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: first_field + id: 1 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "1"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, MatchSecondLayerBool) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: payload + id: 3 + child: + name: f1 + id: 1 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "1"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, MatchSecondLayerByte) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: payload + id: 3 + child: + name: f2 + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "2"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, MatchSecondLayerDouble) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: payload + id: 3 + child: + name: f3 + id: 3 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "3.000000"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, MatchSecondLayerInt16) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: payload + id: 3 + child: + name: f4 + id: 4 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "4"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, MatchSecondLayerInt32) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: payload + id: 3 + child: + name: f5 + id: 5 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "5"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, MatchSecondLayerInt64) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: payload + id: 3 + child: + name: f6 + id: 6 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "6"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, EmptyMethodName) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: "" + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "two"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, DoNotMatchServiceName) { + const std::string request_config_yaml = R"EOF( +request_rules: + - service_name: unknown + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + initializeFilter(request_config_yaml, false); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, DefaultNamespaceTest) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + key: present + on_missing: + key: missing + value: unknown +)EOF"; + initializeFilter(request_config_yaml); + std::map expected = {{"present", "two"}}; + EXPECT_CALL(req_info_, + setDynamicMetadata("envoy.filters.thrift.payload_to_metadata", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, ReplaceValueTest) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + value: bar + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "bar"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, SubstituteValueTest) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + regex_value_rewrite: + pattern: + google_re2: {} + regex: "^(\\w+)$" + substitution: "\\1 cents" + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"present", "two cents"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, NoMatchSubstituteValueTest) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + regex_value_rewrite: + pattern: + google_re2: {} + regex: "^(\\w+)xxxx$" + substitution: "\\1 cents" + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + const std::string value = "do not match"; + std::map expected = {{"present", value}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data, value); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Test empty value doesn't get written to metadata. +TEST_F(PayloadToMetadataTest, SubstituteEmptyValueTest) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + regex_value_rewrite: + pattern: + google_re2: {} + regex: "^hello (\\w+)?.*$" + substitution: "\\1" + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + const std::string value = "hello !!!!"; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data, value); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Test the value gets written as a number. +TEST_F(PayloadToMetadataTest, NumberTypeTest) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: number + type: NUMBER + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + std::map expected = {{"number", 1}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEqNum(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data, "1"); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Test the invalid value won't get written as a number. +TEST_F(PayloadToMetadataTest, BadNumberTypeTest) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: number + type: NUMBER + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data, "invalid"); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Number to number test +TEST_F(PayloadToMetadataTest, NumberToNumberType) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: first_field + id: 1 + on_present: + metadata_namespace: envoy.lb + key: number + type: NUMBER +)EOF"; + + std::map expected = {{"number", 1}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEqNum(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data, "1"); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Number to number with regex test +TEST_F(PayloadToMetadataTest, NumberToNumberTypeWithRegex) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: first_field + id: 1 + on_present: + metadata_namespace: envoy.lb + key: number + regex_value_rewrite: + pattern: + google_re2: {} + regex: "^(\\w+)$" + substitution: "91\\1 " + type: NUMBER +)EOF"; + + std::map expected = {{"number", 911}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEqNum(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data, "1"); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Set configured value when payload is missing. +TEST_F(PayloadToMetadataTest, MissingValueInFirstLayer) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: too_big + id: 100 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + std::map expected = {{"missing", "unknown"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Set configured value when payload is missing in the second layer. +TEST_F(PayloadToMetadataTest, MissingValueInSecondLayer) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: third_field + id: 3 + child: + name: too_big + id: 100 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + std::map expected = {{"missing", "unknown"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Set configured value when payload is missing in the third layer. +TEST_F(PayloadToMetadataTest, MissingValueInThirdLayer) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: third_field + id: 3 + child: + name: f1 + id: 1 + child: + name: unknown + id: 1 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + std::map expected = {{"missing", "unknown"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Perform on_missing while field selector points to a struct. +TEST_F(PayloadToMetadataTest, FieldSelectorPointsToStruct) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: third_field + id: 3 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + std::map expected = {{"missing", "unknown"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Perform on_missing while field selector points to a map. +TEST_F(PayloadToMetadataTest, FieldSelectorPointsToMap) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: third_field + id: 3 + child: + name: f8 + id: 8 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + std::map expected = {{"missing", "unknown"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Set configured value when payload is missing in the second layer. +TEST_F(PayloadToMetadataTest, PointToNonExistSecondLayer) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: third_field + id: 3 + child: + name: too_big + id: 100 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + std::map expected = {{"missing", "unknown"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// on_missing is not executed when payload is present. +TEST_F(PayloadToMetadataTest, NoApplyOnMissingWhenPayloadIsPresent) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// on_present is not executed when payload is missing. +TEST_F(PayloadToMetadataTest, NoApplyOnPresentWhenPayloadIsPresent) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: too_big + id: 100 + on_present: + metadata_namespace: envoy.lb + key: present +)EOF"; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// No payload value does not set any metadata. +TEST_F(PayloadToMetadataTest, EmptyPayloadValue) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + // empty payload on the field + writeMessage(*metadata, data, ""); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +// Payload value too long does not set payload value as metadata. +TEST_F(PayloadToMetadataTest, PayloadValueTooLong) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + on_missing: + metadata_namespace: envoy.lb + key: missing + value: unknown +)EOF"; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + + auto length = MAX_PAYLOAD_VALUE_LEN + 1; + writeMessage(*metadata, data, std::string(length, 'x')); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, MultipleRulesTest) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + - method_name: bar + field_selector: + name: first_field + id: 1 + on_present: + metadata_namespace: envoy.lb + key: method_not_match + - method_name: foo + field_selector: + name: third_field + id: 3 + child: + name: f6 + id: 6 + on_present: + metadata_namespace: envoy.lb + key: six + - method_name: foo + field_selector: + name: third_field + id: 3 + child: + name: f7 + id: 7 + on_present: + metadata_namespace: envoy.lb + key: seven +)EOF"; + + std::map expected = { + {"present", "two"}, {"six", "6"}, {"seven", "seven"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +TEST_F(PayloadToMetadataTest, MultipleRulesInSamePath) { + const std::string request_config_yaml = R"EOF( +request_rules: + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present + - method_name: foo + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: present2 + - method_name: bar + field_selector: + name: second_field + id: 2 + on_present: + metadata_namespace: envoy.lb + key: method_not_match + - method_name: foo + field_selector: + name: third_field + id: 3 + child: + name: f6 + id: 6 + on_present: + metadata_namespace: envoy.lb + key: six + - method_name: bar + field_selector: + name: third_field + id: 3 + child: + name: f6 + id: 6 + on_present: + metadata_namespace: envoy.lb + key: method_not_match_again +)EOF"; + + std::map expected = { + {"present", "two"}, {"present2", "two"}, {"six", "6"}}; + + initializeFilter(request_config_yaml); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + + Buffer::OwnedImpl data; + auto metadata = std::make_shared(); + writeMessage(*metadata, data); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->messageBegin(metadata)); + EXPECT_EQ(ThriftProxy::FilterStatus::Continue, filter_->passthroughData(data)); + filter_->onDestroy(); +} + +} // namespace PayloadToMetadataFilter +} // namespace ThriftFilters +} // namespace Extensions +} // namespace Envoy From ddbd4500e93845389af6191b56ee8b92a06b7089 Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 28 Oct 2022 19:02:02 +0100 Subject: [PATCH 033/112] deps: Bump `aspect_bazel_lib` -> 1.15.0 (#23689) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index c23cd97ac256..43edfef51114 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -136,12 +136,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Aspect Bazel helpers", project_desc = "Base Starlark libraries and basic Bazel rules which are useful for constructing rulesets and BUILD files", project_url = "https://github.com/aspect-build/bazel-lib", - version = "1.12.0", - sha256 = "229a6d65b8b30af0dcaeb77c723b568ef7a2e4ad54dd65168a16992b6b6fe4c7", + version = "1.15.0", + sha256 = "eae670935704ce5f9d050b2c23d426b4ae453458830eebdaac1f11a6a9da150b", strip_prefix = "bazel-lib-{version}", urls = ["https://github.com/aspect-build/bazel-lib/archive/v{version}.tar.gz"], use_category = ["build"], - release_date = "2022-09-16", + release_date = "2022-10-27", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/aspect-build/bazel-lib/blob/v{version}/LICENSE", From 4a16086bbd956f6adf93880f33634f38fcdbe3c1 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Fri, 28 Oct 2022 11:27:04 -0700 Subject: [PATCH 034/112] docs: Update doc string for `server_names` (#23629) Signed-off-by: Arko Dasgupta --- api/envoy/config/listener/v3/listener_components.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/api/envoy/config/listener/v3/listener_components.proto b/api/envoy/config/listener/v3/listener_components.proto index 303b5fb564e3..150a6851d523 100644 --- a/api/envoy/config/listener/v3/listener_components.proto +++ b/api/envoy/config/listener/v3/listener_components.proto @@ -155,6 +155,7 @@ message FilterChainMatch { // will be first matched against ``www.example.com``, then ``*.example.com``, then ``*.com``. // // Note that partial wildcards are not supported, and values like ``*w.example.com`` are invalid. + // The value ``*`` is also not supported, and ``server_names`` should be omitted instead. // // .. attention:: // From 7af7ef6b89c0d34c7dfa60f8ec606a653cd90242 Mon Sep 17 00:00:00 2001 From: Loong Dai Date: Sat, 29 Oct 2022 03:32:17 +0800 Subject: [PATCH 035/112] envoy_reloadable_features_override_request_timeout_by_gateway_timeout deprecation (#23681) Fix #23599 Signed-off-by: Loong --- changelogs/current.yaml | 3 ++ source/common/http/conn_manager_impl.cc | 35 +++---------------- source/common/http/utility.cc | 11 +++--- source/common/runtime/runtime_features.cc | 1 - test/integration/protocol_integration_test.cc | 30 ---------------- 5 files changed, 11 insertions(+), 69 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 1d024a6b2cc8..cdda2c55d1d4 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -41,6 +41,9 @@ removed_config_or_runtime: removed ``envoy.reloadable_features.use_new_codec_wrapper`` and legacy code paths. removed ``envoy.reloadable_features.append_to_accept_content_encoding_only_once`` and legacy code paths. removed ``envoy.reloadable_features.http1_lazy_read_disable`` and legacy code paths. + - area: http + change: | + removed ``envoy.reloadable_features.override_request_timeout_by_gateway_timeout`` and legacy code paths. new_features: - area: generic_proxy diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 06422fa156e9..da099cd82d68 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -770,37 +770,10 @@ void ConnectionManagerImpl::ActiveStream::resetIdleTimer() { void ConnectionManagerImpl::ActiveStream::onIdleTimeout() { connection_manager_.stats_.named_.downstream_rq_idle_timeout_.inc(); - // See below for more information on this early return block. - if (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.override_request_timeout_by_gateway_timeout")) { - filter_manager_.streamInfo().setResponseFlag(StreamInfo::ResponseFlag::StreamIdleTimeout); - sendLocalReply(Http::Utility::maybeRequestTimeoutCode(filter_manager_.remoteDecodeComplete()), - "stream timeout", nullptr, absl::nullopt, - StreamInfo::ResponseCodeDetails::get().StreamIdleTimeout); - return; - } - - // There are 2 issues in the blow code. First, `responseHeaders().has_value()` is not the best - // predicate. `remoteDecodeComplete()` is preferable. Second, `sendLocalReply()` smartly ends the - // stream if any response was pushed to decoder and explicitly `endStream()` is not required. - // - // The above code is expected to resolve both. The original code here before it is fully verified. - // - // TODO(lambdai): delete the block below along with the removal of - // `override_request_timeout_by_gateway_timeout`. - - // If headers have not been sent to the user, send a 408. - if (responseHeaders().has_value()) { - // TODO(htuch): We could send trailers here with an x-envoy timeout header - // or gRPC status code, and/or set H2 RST_STREAM error. - filter_manager_.streamInfo().setResponseCodeDetails( - StreamInfo::ResponseCodeDetails::get().StreamIdleTimeout); - connection_manager_.doEndStream(*this); - } else { - filter_manager_.streamInfo().setResponseFlag(StreamInfo::ResponseFlag::StreamIdleTimeout); - sendLocalReply(Http::Code::RequestTimeout, "stream timeout", nullptr, absl::nullopt, - StreamInfo::ResponseCodeDetails::get().StreamIdleTimeout); - } + filter_manager_.streamInfo().setResponseFlag(StreamInfo::ResponseFlag::StreamIdleTimeout); + sendLocalReply(Http::Utility::maybeRequestTimeoutCode(filter_manager_.remoteDecodeComplete()), + "stream timeout", nullptr, absl::nullopt, + StreamInfo::ResponseCodeDetails::get().StreamIdleTimeout); } void ConnectionManagerImpl::ActiveStream::onRequestTimeout() { diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc index 10c5c12f5c41..498ba56cf96a 100644 --- a/source/common/http/utility.cc +++ b/source/common/http/utility.cc @@ -1135,13 +1135,10 @@ bool Utility::isSafeRequest(const Http::RequestHeaderMap& request_headers) { } Http::Code Utility::maybeRequestTimeoutCode(bool remote_decode_complete) { - return remote_decode_complete && - Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.override_request_timeout_by_gateway_timeout") - ? Http::Code::GatewayTimeout - // Http::Code::RequestTimeout is more expensive because HTTP1 client cannot use the - // connection any more. - : Http::Code::RequestTimeout; + return remote_decode_complete ? Http::Code::GatewayTimeout + // Http::Code::RequestTimeout is more expensive because HTTP1 client + // cannot use the connection any more. + : Http::Code::RequestTimeout; } } // namespace Http diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 8084e5db25df..1eddea855118 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -57,7 +57,6 @@ RUNTIME_GUARD(envoy_reloadable_features_lua_respond_with_send_local_reply); RUNTIME_GUARD(envoy_reloadable_features_no_delay_close_for_upgrades); RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name); RUNTIME_GUARD(envoy_reloadable_features_original_dst_rely_on_idle_timeout); -RUNTIME_GUARD(envoy_reloadable_features_override_request_timeout_by_gateway_timeout); RUNTIME_GUARD(envoy_reloadable_features_postpone_h3_client_connect_to_next_loop); RUNTIME_GUARD(envoy_reloadable_features_quic_defer_send_in_response_to_packet); RUNTIME_GUARD(envoy_reloadable_features_skip_delay_close); diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 62c107b1e83d..7201244e34e4 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -2666,36 +2666,6 @@ TEST_P(ProtocolIntegrationTest, MaxStreamTimeoutWhenRequestIsNotComplete) { EXPECT_EQ("504", response->headers().getStatusValue()); } -// Test case above except disabling runtime guard "override_request_timeout_by_gateway_timeout". -// Verify the old behavior is reverted by disabling the runtime guard. -TEST_P(ProtocolIntegrationTest, MaxStreamTimeoutWhenRequestIsNotCompleteRuntimeDisabled) { - config_helper_.setDownstreamMaxStreamDuration(std::chrono::milliseconds(500)); - - config_helper_.addRuntimeOverride( - "envoy.reloadable_features.override_request_timeout_by_gateway_timeout", "false"); - autonomous_upstream_ = false; - - initialize(); - codec_client_ = makeHttpConnection(lookupPort("http")); - - // The request is not header only. Envoy is expecting more data to end the request. - auto encoder_decoder = - codec_client_->startRequest(default_request_headers_, /*header_only_request=*/true); - request_encoder_ = &encoder_decoder.first; - auto response = std::move(encoder_decoder.second); - - ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); - ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); - ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); - - test_server_->waitForCounterGe("http.config_test.downstream_rq_max_duration_reached", 1); - ASSERT_TRUE(response->waitForEndStream()); - - EXPECT_TRUE(upstream_request_->complete()); - ASSERT_TRUE(response->complete()); - EXPECT_EQ("408", response->headers().getStatusValue()); -} - TEST_P(DownstreamProtocolIntegrationTest, MaxRequestsPerConnectionReached) { config_helper_.setDownstreamMaxRequestsPerConnection(2); initialize(); From ff6371055532f6b0af47a98306dfc7abc28d6eac Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Fri, 28 Oct 2022 16:03:53 -0400 Subject: [PATCH 036/112] remove envoy.reloadable_features.support_locality_update_on_eds_cluster_endpoints (#23673) Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: inline Fixes #23603 Signed-off-by: Alyssa Wilk --- changelogs/current.yaml | 3 ++ source/common/runtime/runtime_features.cc | 1 - source/common/upstream/upstream_impl.cc | 12 ++--- test/common/upstream/eds_test.cc | 65 ----------------------- 4 files changed, 7 insertions(+), 74 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index cdda2c55d1d4..23ee1c0ecb6c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -21,6 +21,9 @@ bug_fixes: removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` + - area: eds + change: | + removed ``envoy.reloadable_features.support_locality_update_on_eds_cluster_endpoints`` and legacy code paths. - area: listener change: | removed ``envoy.reloadable_features.strict_check_on_ipv4_compat`` and legacy code paths. diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 1eddea855118..a186362787a9 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -61,7 +61,6 @@ RUNTIME_GUARD(envoy_reloadable_features_postpone_h3_client_connect_to_next_loop) RUNTIME_GUARD(envoy_reloadable_features_quic_defer_send_in_response_to_packet); RUNTIME_GUARD(envoy_reloadable_features_skip_delay_close); RUNTIME_GUARD(envoy_reloadable_features_skip_dns_lookup_for_proxied_requests); -RUNTIME_GUARD(envoy_reloadable_features_support_locality_update_on_eds_cluster_endpoints); RUNTIME_GUARD(envoy_reloadable_features_test_feature_true); RUNTIME_GUARD(envoy_reloadable_features_tls_async_cert_validation); RUNTIME_GUARD(envoy_reloadable_features_top_level_ecds_stats); diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 30bfaceeb627..b6bc7cc4f8e3 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1872,14 +1872,10 @@ bool BaseDynamicClusterImpl::updateDynamicHostList( (health_checker_ != nullptr && existing_host_found && *existing_host->second->healthCheckAddress() != *host->healthCheckAddress()); bool locality_changed = false; - if (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.support_locality_update_on_eds_cluster_endpoints")) { - locality_changed = - (existing_host_found && - (!LocalityEqualTo()(host->locality(), existing_host->second->locality()))); - if (locality_changed) { - hosts_with_updated_locality_for_current_priority.emplace(existing_host->first); - } + locality_changed = (existing_host_found && + (!LocalityEqualTo()(host->locality(), existing_host->second->locality()))); + if (locality_changed) { + hosts_with_updated_locality_for_current_priority.emplace(existing_host->first); } const bool skip_inplace_host_update = health_check_address_changed || locality_changed; diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index 7926b191352d..88fb82a0c9b2 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -1548,71 +1548,6 @@ TEST_F(EdsTest, EndpointLocalityUpdated) { } } -// Validate that onConfigUpdate() does not update the endpoint locality if fix for the issue, -// https://github.com/envoyproxy/envoy/issues/12392, is disabled. -// Unlike EndpointLocalityUpdated, runtime feature flag is disabled this time and then it is -// verified that locality update does not happen on eds cluster endpoints. -TEST_F(EdsTest, EndpointLocalityNotUpdatedIfFixDisabled) { - runtime_.mergeValues( - {{"envoy.reloadable_features.support_locality_update_on_eds_cluster_endpoints", "false"}}); - envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment; - cluster_load_assignment.set_cluster_name("fare"); - auto* endpoints = cluster_load_assignment.add_endpoints(); - auto* locality = endpoints->mutable_locality(); - locality->set_region("oceania"); - locality->set_zone("hello"); - locality->set_sub_zone("world"); - - { - auto* endpoint_address = endpoints->add_lb_endpoints() - ->mutable_endpoint() - ->mutable_address() - ->mutable_socket_address(); - endpoint_address->set_address("1.2.3.4"); - endpoint_address->set_port_value(80); - } - { - auto* endpoint_address = endpoints->add_lb_endpoints() - ->mutable_endpoint() - ->mutable_address() - ->mutable_socket_address(); - endpoint_address->set_address("2.3.4.5"); - endpoint_address->set_port_value(80); - } - - initialize(); - doOnConfigUpdateVerifyNoThrow(cluster_load_assignment); - EXPECT_TRUE(initialized_); - - auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts(); - EXPECT_EQ(hosts.size(), 2); - for (int i = 0; i < 2; ++i) { - EXPECT_EQ(0, hosts[i]->priority()); - const auto& locality = hosts[i]->locality(); - EXPECT_EQ("oceania", locality.region()); - EXPECT_EQ("hello", locality.zone()); - EXPECT_EQ("world", locality.sub_zone()); - } - EXPECT_EQ(nullptr, cluster_->prioritySet().hostSetsPerPriority()[0]->localityWeights()); - - // Update locality now - locality->set_region("space"); - locality->set_zone("station"); - locality->set_sub_zone("mars"); - doOnConfigUpdateVerifyNoThrow(cluster_load_assignment); - - // runtime flag is disabled, verify that locality does not get updated - auto& updatedHosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts(); - EXPECT_EQ(updatedHosts.size(), 2); - for (int i = 0; i < 2; ++i) { - EXPECT_EQ(0, updatedHosts[i]->priority()); - const auto& locality = updatedHosts[i]->locality(); - EXPECT_EQ("oceania", locality.region()); - EXPECT_EQ("hello", locality.zone()); - EXPECT_EQ("world", locality.sub_zone()); - } -} - // Validate that onConfigUpdate() does not propagate locality weights to the host set when // locality weighted balancing isn't configured and the cluster does not use LB policy extensions. TEST_F(EdsTest, EndpointLocalityWeightsIgnored) { From b3e59bda3e1ae96640f84ec3d4d32f632b378d37 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 28 Oct 2022 13:04:21 -0700 Subject: [PATCH 037/112] bazel: fix more config_setting visibility (#23709) Fixes https://github.com/envoyproxy/envoy/issues/23390 This is an edge case, upstream issue: https://github.com/bazelbuild/bazel-skylib/issues/404 Signed-off-by: Keith Smiley --- bazel/BUILD | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bazel/BUILD b/bazel/BUILD index 31bb1ea4016b..c8bb5f64da90 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -304,7 +304,8 @@ config_setting( bool_flag( name = "http3", build_setting_default = True, - visibility = ["//visibility:private"], + # TODO(keith): make private again https://github.com/bazelbuild/bazel-skylib/issues/404 + # visibility = ["//visibility:private"], ) config_setting( @@ -312,7 +313,8 @@ config_setting( flag_values = { ":http3": "False", }, - visibility = ["//visibility:private"], + # TODO(keith): make private again https://github.com/bazelbuild/bazel-skylib/issues/404 + # visibility = ["//visibility:private"], ) selects.config_setting_group( @@ -331,7 +333,8 @@ config_setting( config_setting( name = "disable_hot_restart_setting", values = {"define": "hot_restart=disabled"}, - visibility = ["//visibility:private"], + # TODO(keith): make private again https://github.com/bazelbuild/bazel-skylib/issues/404 + # visibility = ["//visibility:private"], ) selects.config_setting_group( From ba81ae537e722ad99906599f5d3ad367ca854433 Mon Sep 17 00:00:00 2001 From: phlax Date: Sat, 29 Oct 2022 00:24:02 +0100 Subject: [PATCH 038/112] deps: Bump `rules_python` -> 0.13.0 (#23690) Signed-off-by: Ryan Northey --- bazel/dependency_imports.bzl | 2 ++ bazel/python_dependencies.bzl | 6 +++--- bazel/repository_locations.bzl | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index b62d3cf28443..64190a5946a1 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -8,6 +8,7 @@ load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains") load("@proxy_wasm_rust_sdk//bazel:dependencies.bzl", "proxy_wasm_rust_sdk_dependencies") load("@base_pip3//:requirements.bzl", pip_dependencies = "install_deps") +load("@fuzzing_pip3//:requirements.bzl", pip_fuzzing_dependencies = "install_deps") load("@emsdk//:emscripten_deps.bzl", "emscripten_deps") load("@com_github_aignas_rules_shellcheck//:deps.bzl", "shellcheck_dependencies") load("@aspect_bazel_lib//lib:repositories.bzl", "register_jq_toolchains", "register_yq_toolchains") @@ -27,6 +28,7 @@ def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, y gazelle_dependencies() apple_rules_dependencies() pip_dependencies() + pip_fuzzing_dependencies() rules_pkg_dependencies() rules_rust_dependencies() rust_register_toolchains( diff --git a/bazel/python_dependencies.bzl b/bazel/python_dependencies.bzl index d9dfb14a9b6b..a5c3283d0a25 100644 --- a/bazel/python_dependencies.bzl +++ b/bazel/python_dependencies.bzl @@ -9,7 +9,7 @@ def envoy_python_dependencies(): extra_pip_args = ["--require-hashes"], ) - # These need to use `pip_install` + # TODO(phlax): switch to `pip_parse` pip_install( # Note: dev requirements do *not* check hashes python_interpreter_target = interpreter, @@ -17,9 +17,9 @@ def envoy_python_dependencies(): requirements = "@envoy//tools/dev:requirements.txt", ) - pip_install( + pip_parse( name = "fuzzing_pip3", python_interpreter_target = interpreter, - requirements = "@rules_fuzzing//fuzzing:requirements.txt", + requirements_lock = "@rules_fuzzing//fuzzing:requirements.txt", extra_pip_args = ["--require-hashes"], ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 43edfef51114..0da3f798ac58 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -909,9 +909,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Python rules for Bazel", project_desc = "Bazel rules for the Python language", project_url = "https://github.com/bazelbuild/rules_python", - version = "0.12.0", - sha256 = "b593d13bb43c94ce94b483c2858e53a9b811f6f10e1e0eedc61073bd90e58d9c", - release_date = "2022-08-29", + version = "0.13.0", + sha256 = "8c8fe44ef0a9afc256d1e75ad5f448bb59b81aba149b8958f02f7b3a98f5d9b4", + release_date = "2022-09-25", strip_prefix = "rules_python-{version}", urls = ["https://github.com/bazelbuild/rules_python/archive/{version}.tar.gz"], use_category = ["build"], From eff51a344b77068081332ef83e6b79dc87dec192 Mon Sep 17 00:00:00 2001 From: Amit Saha <76932416+asaha123@users.noreply.github.com> Date: Mon, 31 Oct 2022 19:42:17 +1100 Subject: [PATCH 039/112] Example for route mirroring policy (#23336) Signed-off-by: Amit Saha --- docs/root/start/sandboxes/index.rst | 1 + docs/root/start/sandboxes/route-mirror.rst | 188 +++++++++++++++++++++ examples/route-mirror/Dockerfile-envoy | 5 + examples/route-mirror/README.md | 2 + examples/route-mirror/docker-compose.yaml | 50 ++++++ examples/route-mirror/front-envoy.yaml | 90 ++++++++++ examples/route-mirror/service.py | 16 ++ examples/route-mirror/verify.sh | 36 ++++ 8 files changed, 388 insertions(+) create mode 100644 docs/root/start/sandboxes/route-mirror.rst create mode 100644 examples/route-mirror/Dockerfile-envoy create mode 100644 examples/route-mirror/README.md create mode 100644 examples/route-mirror/docker-compose.yaml create mode 100644 examples/route-mirror/front-envoy.yaml create mode 100644 examples/route-mirror/service.py create mode 100755 examples/route-mirror/verify.sh diff --git a/docs/root/start/sandboxes/index.rst b/docs/root/start/sandboxes/index.rst index 96408466d9cc..dce2abe43e4c 100644 --- a/docs/root/start/sandboxes/index.rst +++ b/docs/root/start/sandboxes/index.rst @@ -69,6 +69,7 @@ The following sandboxes are available: postgres rbac redis + route-mirror skywalking tls-inspector tls-sni diff --git a/docs/root/start/sandboxes/route-mirror.rst b/docs/root/start/sandboxes/route-mirror.rst new file mode 100644 index 000000000000..428cf368059d --- /dev/null +++ b/docs/root/start/sandboxes/route-mirror.rst @@ -0,0 +1,188 @@ +.. _install_sandboxes_route_mirror: + +Route mirroring policies +======================== + +.. sidebar:: Requirements + + .. include:: _include/docker-env-setup-link.rst + +This simple example demonstrates Envoy's request mirroring capability using +`request mirror policies `__. + +Incoming requests are received by ``envoy-front-proxy`` service. + +Requests for the path ``/service/1`` are statically mirrored. + +Each request is handled by the ``service1`` cluster, and in addition, forwarded to +the ``service1-mirror`` cluster: + +.. literalinclude:: _include/route-mirror/front-envoy.yaml + :language: yaml + :lines: 16-34 + :linenos: + :emphasize-lines: 6-11 + :caption: Envoy configuration with static route mirror policy :download:`front-envoy.yaml <_include/route-mirror/front-envoy.yaml>` + +Requests for the path ``/service/2`` are dynamically mirrored according to the presence and value of +the ``x-mirror-cluster`` header. + +All reqests for this path are forwarded to the ``service2`` cluster, and are also mirrored +to the cluster named in the header. + +For example, if we send a request with the header ``x-mirror-cluster: service2-mirror``, +the request will be forwarded to the ``service2-mirror`` cluster. + +.. literalinclude:: _include/route-mirror/front-envoy.yaml + :language: yaml + :lines: 16-34 + :linenos: + :emphasize-lines: 12-17 + :caption: Envoy configuration with header based route mirror policy :download:`front-envoy.yaml <_include/route-mirror/front-envoy.yaml>` + + +.. warning:: + + Allowing a request header to determine the cluster that the request is mirrored to is most useful in + a trusted environment. + + For example, a downstream Envoy instance (or other application acting as a proxy) might + automatically add this header to requests for processing by an upstream Envoy instance + configured with request mirror policies. + + If you allow dynamic mirroring according to request header, you may wish to restrict which requests + can set or proxy the header. + +.. note:: + + Envoy will only return the response it receives from the primary cluster to the client. + + For this example, responses from ``service1`` and ``service2`` clusters will be sent + to the client. A response returned by the ``service1-mirror`` or the ``service2-mirror`` + cluster is not sent back to the client. + + This also means that any problems or latency in request processing in the mirror cluster + don't affect the response received by the client. + +Step 1: Build the sandbox +************************* + +Change to the ``examples/route-mirror`` directory. + +.. code-block:: console + + $ pwd + envoy/examples/route-mirror + $ docker-compose build + $ docker-compose up -d + $ docker-compose ps + NAME COMMAND SERVICE STATUS PORTS + route-mirror-envoy-front-proxy-1 "/docker-entrypoint.…" envoy-front-proxy running 0.0.0.0:10000->10000/tcp, :::10000->10000/tcp + route-mirror-service1-1 "python3 /code/servi…" service1 running (healthy) + route-mirror-service1-mirror-1 "python3 /code/servi…" service1-mirror running (healthy) + route-mirror-service2-1 "python3 /code/servi…" service2 running (healthy) + route-mirror-service2-mirror-1 "python3 /code/servi…" service2-mirror running (healthy) + +Step 2: Make a request to the statically mirrored route +******************************************************* + +Let's send a request to the ``envoy-front-proxy`` service which forwards the request to +``service1`` and also sends the request to the service 1 mirror, ``service1-mirror``. + +.. code-block:: console + + $ curl localhost:10000/service/1 + Hello from behind Envoy (service 1)! + +Step 3: View logs for the statically mirrored request +***************************************************** + +The logs from the ``service1`` and ``service1-mirror`` services show that +both the ``service1`` and ``service1-mirror`` services received the request made +in Step 2. + +You can also see that for the request to the ``service1-mirror`` +service, the ``Host`` header was modified by Envoy to have a ``-shadow`` suffix +in the hostname. + +.. code-block:: console + + $ docker-compose logs service1 + ... + Host: localhost:10000 + 192.168.80.6 - - [06/Oct/2022 03:56:22] "GET /service/1 HTTP/1.1" 200 - + + $ docker-compose logs service1-mirror + ... + Host: localhost-shadow:10000 + 192.168.80.6 - - [06/Oct/2022 03:56:22] "GET /service/1 HTTP/1.1" 200 - + + +Step 4: Make a request to the route mirrored by request header +************************************************************** + +In this step, we will see a demonstration where the request specifies via a header, ``x-mirror-cluster``, +the cluster that envoy will mirror the request to. + +Let's send a request to the ``envoy-front-proxy`` service which forwards the request to +``service2`` and also mirrors the request to the cluster named, ``service2-mirror``. + +.. code-block:: console + + $ curl --header "x-mirror-cluster: service2-mirror" localhost:10000/service/2 + Hello from behind Envoy (service 2)! + + +Step 5: View logs for the request mirrored by request header +************************************************************ + +The logs show that both the ``service2`` and ``service2-mirror`` services +got the request. + +.. code-block:: console + + $ docker-compose logs service2 + ... + Host: localhost:10000 + 192.168.80.6 - - [06/Oct/2022 03:56:22] "GET /service/2 HTTP/1.1" 200 - + + $ docker-compose logs service2-mirror + ... + Host: localhost-shadow:10000 + 192.168.80.6 - - [06/Oct/2022 03:56:22] "GET /service/2 HTTP/1.1" 200 - + +You can also see that for the request to the ``service2-mirror`` service, the +``Host`` header was modified by Envoy to have a ``-shadow`` suffix in the +hostname. + +Step 6: Missing or invalid cluster name in request header +********************************************************* + +If you do not specify the ``x-mirror-cluster`` in the request to ``service2``, +or specify an unknown cluster, the request will not be mirrored but will be +handled in the normal way. + +.. code-block:: console + + $ curl localhost:10000/service/2 + Hello from behind Envoy (service 2)! + + $ curl --header "x-mirror-cluster: service2-mirror-non-existent" localhost:10000/service/2 + Hello from behind Envoy (service 2)! + +View the logs for ``service2`` and ``service2-mirror`` services. + +.. code-block:: console + + $ docker-compose logs service2 + ... + Host: localhost:10000 + 192.168.80.6 - - [06/Oct/2022 03:56:22] "GET /service/2 HTTP/1.1" 200 - + + $ docker-compose logs service2-mirror + # No new logs + +.. seealso:: + + :ref:`Envoy request mirror policy ` + Learn more Envoy's request mirroring policy. diff --git a/examples/route-mirror/Dockerfile-envoy b/examples/route-mirror/Dockerfile-envoy new file mode 100644 index 000000000000..67ab98915d01 --- /dev/null +++ b/examples/route-mirror/Dockerfile-envoy @@ -0,0 +1,5 @@ +FROM envoyproxy/envoy-dev:latest + +COPY ./front-envoy.yaml /etc/front-envoy.yaml +RUN chmod go+r /etc/front-envoy.yaml +CMD ["/usr/local/bin/envoy", "-c", "/etc/front-envoy.yaml", "--service-cluster", "front-proxy"] diff --git a/examples/route-mirror/README.md b/examples/route-mirror/README.md new file mode 100644 index 000000000000..644009479424 --- /dev/null +++ b/examples/route-mirror/README.md @@ -0,0 +1,2 @@ +To learn about this sandbox and for instructions on how to run it please head over +to the [envoy docs](https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/route_mirror.html) diff --git a/examples/route-mirror/docker-compose.yaml b/examples/route-mirror/docker-compose.yaml new file mode 100644 index 000000000000..3470c72e58ec --- /dev/null +++ b/examples/route-mirror/docker-compose.yaml @@ -0,0 +1,50 @@ +version: "3.8" +services: + + envoy-front-proxy: + build: + context: . + dockerfile: Dockerfile-envoy + ports: + - "${PORT_PROXY:-10000}:10000" + depends_on: + service1: + condition: service_healthy + service1-mirror: + condition: service_healthy + service2: + condition: service_healthy + service2-mirror: + condition: service_healthy + + service1: + build: + context: ../shared/flask + volumes: + - ./service.py:/code/service.py + environment: + - SERVICE_NAME=1 + + service1-mirror: + build: + context: ../shared/flask + volumes: + - ./service.py:/code/service.py + environment: + - SERVICE_NAME=1 + + service2: + build: + context: ../shared/flask + volumes: + - ./service.py:/code/service.py + environment: + - SERVICE_NAME=2 + + service2-mirror: + build: + context: ../shared/flask + volumes: + - ./service.py:/code/service.py + environment: + - SERVICE_NAME=2 diff --git a/examples/route-mirror/front-envoy.yaml b/examples/route-mirror/front-envoy.yaml new file mode 100644 index 000000000000..f47e42e68dc3 --- /dev/null +++ b/examples/route-mirror/front-envoy.yaml @@ -0,0 +1,90 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 10000 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: backend + domains: + - "*" + routes: + - match: + prefix: "/service/1" + route: + cluster: service1 + request_mirror_policies: + - cluster: "service1-mirror" + - match: + prefix: "/service/2" + route: + cluster: service2 + request_mirror_policies: + - cluster_header: "x-mirror-cluster" + + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + clusters: + - name: service1 + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service1 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: service1 + port_value: 8080 + + - name: service1-mirror + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service1-mirror + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: service1-mirror + port_value: 8080 + + - name: service2 + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service2 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: service2 + port_value: 8080 + + - name: service2-mirror + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service2-mirror + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: service2-mirror + port_value: 8080 diff --git a/examples/route-mirror/service.py b/examples/route-mirror/service.py new file mode 100644 index 000000000000..3c97765ce931 --- /dev/null +++ b/examples/route-mirror/service.py @@ -0,0 +1,16 @@ +import os + +from flask import Flask, request +from flask.helpers import send_from_directory + +app = Flask(__name__) + + +@app.route(f"/service/{os.environ['SERVICE_NAME']}") +def get_service(): + print(f"Host: {request.headers.get('Host')}", flush=True) + return f"Hello from behind Envoy (service {os.environ['SERVICE_NAME']})!\n" + + +if __name__ == "__main__": + app.run(host='0.0.0.0', port=8080) diff --git a/examples/route-mirror/verify.sh b/examples/route-mirror/verify.sh new file mode 100755 index 000000000000..b27bcb8d15d2 --- /dev/null +++ b/examples/route-mirror/verify.sh @@ -0,0 +1,36 @@ +#!/bin/bash -e + +export NAME=route-mirroring +export PORT_PROXY="${FRONT_PROXY_PORT_PROXY:-11820}" + +# shellcheck source=examples/verify-common.sh +. "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh" + +run_log "Make a request to the statically mirrored route" +responds_with "Hello from behind Envoy (service 1)!" "http://localhost:${PORT_PROXY}/service/1" + +run_log "View logs for the request mirrored by request header" +docker-compose logs service1 | grep --quiet "Host: localhost:${PORT_PROXY}" +docker-compose logs service1-mirror | grep --quiet "Host: localhost-shadow:${PORT_PROXY}" +docker-compose logs service1-mirror | grep --quiet GET + +run_log "Make a request to the route mirrored by request header" +responds_with \ + "Hello from behind Envoy (service 2)!" \ + "http://localhost:${PORT_PROXY}/service/2" \ + --header 'x-mirror-cluster: service2-mirror' + +run_log "View logs for the request mirrored by request header" +docker-compose logs service2 | grep --quiet "Host: localhost:${PORT_PROXY}" +docker-compose logs service2-mirror | grep --quiet "Host: localhost-shadow:${PORT_PROXY}" +docker-compose logs service2-mirror | grep --quiet GET + +run_log "Missing or invalid cluster name in request header" +responds_with \ + "Hello from behind Envoy (service 2)!" \ + "http://localhost:${PORT_PROXY}/service/2" +responds_with \ + "Hello from behind Envoy (service 2)!" \ + "http://localhost:${PORT_PROXY}/service/2" \ + --header 'x-mirror-cluster: service2-mirror-non-existent' +docker-compose logs service2-mirror | grep -c GET | grep --quiet 1 From dbee14248706204bbdd80f5b78ad479476a2de6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:05:50 +0000 Subject: [PATCH 040/112] build(deps): bump slack-sdk from 3.19.1 to 3.19.2 in /.github/actions/pr_notifier (#23711) build(deps): bump slack-sdk in /.github/actions/pr_notifier Bumps [slack-sdk](https://github.com/slackapi/python-slack-sdk) from 3.19.1 to 3.19.2. - [Release notes](https://github.com/slackapi/python-slack-sdk/releases) - [Changelog](https://github.com/slackapi/python-slack-sdk/blob/main/docs-v2/changelog.html) - [Commits](https://github.com/slackapi/python-slack-sdk/compare/v3.19.1...v3.19.2) --- updated-dependencies: - dependency-name: slack-sdk dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/actions/pr_notifier/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/pr_notifier/requirements.txt b/.github/actions/pr_notifier/requirements.txt index e27f40f783a5..313a70828c5b 100644 --- a/.github/actions/pr_notifier/requirements.txt +++ b/.github/actions/pr_notifier/requirements.txt @@ -111,9 +111,9 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via pynacl -slack-sdk==3.19.1 \ - --hash=sha256:5db78a3ed3c58b3b0a2bdbe40bac43c7b00edcbc12ff6a7778ac5cbe3a3a2899 \ - --hash=sha256:a0bafb9e53ffbc408a8b5a42443525aac2c0c7347223d9373dd9d41cf90c9343 +slack-sdk==3.19.2 \ + --hash=sha256:336365512ee8620a7227c6780af28d3a69db3387653fbeee156bc391cbbdbf47 \ + --hash=sha256:8b36a0c1cd99426df1e7dea9ad55838cafe9de46db88a4c9e4ee787da718a00a # via -r requirements.in urllib3==1.26.6 \ --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \ From 0085ea299b49ac42bb54b4dfb899c94d3588a413 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:06:01 +0000 Subject: [PATCH 041/112] build(deps): bump golang from `f3e6836` to `e4dcdac` in /examples/ext_authz/auth/grpc-service (#23716) build(deps): bump golang in /examples/ext_authz/auth/grpc-service Bumps golang from `f3e6836` to `e4dcdac`. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/ext_authz/auth/grpc-service/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ext_authz/auth/grpc-service/Dockerfile b/examples/ext_authz/auth/grpc-service/Dockerfile index 7986a37097ed..ea4381962dac 100644 --- a/examples/ext_authz/auth/grpc-service/Dockerfile +++ b/examples/ext_authz/auth/grpc-service/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:alpine@sha256:f3e683657ddf73726b5717c2ff80cdcd9e9efb7d81f77e4948fada9a10dc7257 AS builder +FROM golang:alpine@sha256:e4dcdac3ed37d8c2b3b8bcef2909573b2ad9c2ab53ba53c608909e8b89ccee36 AS builder RUN apk --no-cache add make COPY . /app From 0f5c4b33f42ff2addcf641f797b0cf66eaff95dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:06:16 +0000 Subject: [PATCH 042/112] build(deps): bump node from `5a7b677` to `1a04e2e` in /examples/ext_authz/auth/http-service (#23715) build(deps): bump node in /examples/ext_authz/auth/http-service Bumps node from `5a7b677` to `1a04e2e`. --- updated-dependencies: - dependency-name: node dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/ext_authz/auth/http-service/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ext_authz/auth/http-service/Dockerfile b/examples/ext_authz/auth/http-service/Dockerfile index 6a014e395279..9890e3a0ba3f 100644 --- a/examples/ext_authz/auth/http-service/Dockerfile +++ b/examples/ext_authz/auth/http-service/Dockerfile @@ -1,4 +1,4 @@ -FROM node:alpine@sha256:5a7b6772549bfbb856769f9e3d090812450399a746654a8b89a80b2026591902 +FROM node:alpine@sha256:1a04e2ec39cc0c3a9657c1d6f8291ea2f5ccadf6ef4521dec946e522833e87ea COPY . /app CMD ["node", "/app/http-service/server"] From 6e45fdc84227e4df8f2215e83f6f34dfe6518b61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:06:33 +0000 Subject: [PATCH 043/112] build(deps): bump golang from `01b565b` to `8b9971c` in /examples/grpc-bridge/server (#23714) build(deps): bump golang in /examples/grpc-bridge/server Bumps golang from `01b565b` to `8b9971c`. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/grpc-bridge/server/Dockerfile b/examples/grpc-bridge/server/Dockerfile index c7cfcd4c62bf..493de4907eda 100644 --- a/examples/grpc-bridge/server/Dockerfile +++ b/examples/grpc-bridge/server/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.2-bullseye@sha256:01b565b5e745c650b1893c9a1ba6d3966e5ae4a72ec96595c5a7c54e04d96c05 as builder +FROM golang:1.19.2-bullseye@sha256:8b9971c37678d2c4c04e7dd4e430baec049647d72ed97bf5e6a41ef8e77e74a5 as builder WORKDIR /build From f52f99e8617ea534e4d074533c3879d7d51d8d14 Mon Sep 17 00:00:00 2001 From: Timo Haas Date: Mon, 31 Oct 2022 13:55:59 +0100 Subject: [PATCH 044/112] outh2: passthrough of authorization header when filter is matched (#23489) Authorization header sanitation altered headers before the passthrough evaluation. Therefore, move of passthrough header matching to the beginning if the header evaluation, so the unaltered request is passed through. Introduces a new metric: oauth_passthrough counter. Signed-off-by: Timo Haas --- changelogs/current.yaml | 6 ++++ .../http/http_filters/oauth2_filter.rst | 3 +- source/common/runtime/runtime_features.cc | 1 + .../extensions/filters/http/oauth2/filter.cc | 32 ++++++++++++----- .../extensions/filters/http/oauth2/filter.h | 1 + test/extensions/filters/http/oauth2/BUILD | 1 + .../filters/http/oauth2/filter_test.cc | 35 +++++++++++++++++++ 7 files changed, 69 insertions(+), 10 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 23ee1c0ecb6c..611324b94cb5 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -8,6 +8,9 @@ minor_behavior_changes: - area: cache_filter change: | add a completion callback to updateHeaders interface. Any external cache implementations will need to update to match this new interface. See changes to simple_http_cache in PR#23666 for example. +- area: oauth2 + change: | + Requests which match the passthrough header now have their own metric ``oauth_passthrough`` and aren't included in ``oauth_success`` anymore. bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* @@ -17,6 +20,9 @@ bug_fixes: - area: router change: | fixed a bug that incorrectly rewrote the path when using ``regex_rewrite`` for redirects matched on prefix. +- area: oauth2 + change: | + fixed a bug when passthrough header was matched, envoy would always remove the authorization header. This behavioral change can be temporarily reverted by setting runtime guard ``envoy.reloadable_features.oauth_header_passthrough_fix`` to false. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/docs/root/configuration/http/http_filters/oauth2_filter.rst b/docs/root/configuration/http/http_filters/oauth2_filter.rst index 3bf85dc07145..b36bf29edf24 100644 --- a/docs/root/configuration/http/http_filters/oauth2_filter.rst +++ b/docs/root/configuration/http/http_filters/oauth2_filter.rst @@ -229,7 +229,7 @@ sending the user to the configured auth endpoint. :ref:`pass_through_matcher ` provides an interface for users to provide specific header matching criteria such that, when applicable, the OAuth flow is entirely skipped. -When this occurs, the ``oauth_success`` metric is still incremented. +When this occurs, the ``oauth_passthrough`` metric is incremented but ``success`` is not. Generally, allowlisting is inadvisable from a security standpoint. @@ -243,5 +243,6 @@ The OAuth2 filter outputs statistics in the *.* namespace. :widths: 1, 1, 2 oauth_failure, Counter, Total requests that were denied. + oauth_passthrough, Counter, Total request that matched a passthrough header. oauth_success, Counter, Total requests that were allowed. oauth_unauthorization_rq, Counter, Total unauthorized requests. diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index a186362787a9..a70095769234 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -56,6 +56,7 @@ RUNTIME_GUARD(envoy_reloadable_features_local_ratelimit_match_all_descriptors); RUNTIME_GUARD(envoy_reloadable_features_lua_respond_with_send_local_reply); RUNTIME_GUARD(envoy_reloadable_features_no_delay_close_for_upgrades); RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name); +RUNTIME_GUARD(envoy_reloadable_features_oauth_header_passthrough_fix); RUNTIME_GUARD(envoy_reloadable_features_original_dst_rely_on_idle_timeout); RUNTIME_GUARD(envoy_reloadable_features_postpone_h3_client_connect_to_next_loop); RUNTIME_GUARD(envoy_reloadable_features_quic_defer_send_in_response_to_packet); diff --git a/source/extensions/filters/http/oauth2/filter.cc b/source/extensions/filters/http/oauth2/filter.cc index 6bf9d51dd313..d861b24ed477 100644 --- a/source/extensions/filters/http/oauth2/filter.cc +++ b/source/extensions/filters/http/oauth2/filter.cc @@ -19,6 +19,7 @@ #include "source/common/http/message_impl.h" #include "source/common/http/utility.h" #include "source/common/protobuf/utility.h" +#include "source/common/runtime/runtime_features.h" #include "absl/strings/escaping.h" #include "absl/strings/match.h" @@ -221,12 +222,25 @@ const std::string& OAuth2Filter::bearerPrefix() const { /** * primary cases: - * 1) user is signing out - * 2) /_oauth redirect - * 3) user is authorized - * 4) user is unauthorized + * 1) pass through header is matching + * 2) user is signing out + * 3) /_oauth redirect + * 4) user is authorized + * 5) user is unauthorized */ Http::FilterHeadersStatus OAuth2Filter::decodeHeaders(Http::RequestHeaderMap& headers, bool) { + // Skip Filter and continue chain if a Passthrough header is matching + // Must be done before the sanitation of the authorization header, + // otherwise the authorization header might be altered or removed + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.oauth_header_passthrough_fix")) { + for (const auto& matcher : config_->passThroughMatchers()) { + if (matcher.matchesHeaders(headers)) { + config_->stats().oauth_passthrough_.inc(); + return Http::FilterHeadersStatus::Continue; + } + } + } + // Sanitize the Authorization header, since we have no way to validate its content. Also, // if token forwarding is enabled, this header will be set based on what is on the HMAC cookie // before forwarding the request upstream. @@ -376,13 +390,13 @@ bool OAuth2Filter::canSkipOAuth(Http::RequestHeaderMap& headers) const { } return true; } - - for (const auto& matcher : config_->passThroughMatchers()) { - if (matcher.matchesHeaders(headers)) { - return true; + if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.oauth_header_passthrough_fix")) { + for (const auto& matcher : config_->passThroughMatchers()) { + if (matcher.matchesHeaders(headers)) { + return true; + } } } - return false; } diff --git a/source/extensions/filters/http/oauth2/filter.h b/source/extensions/filters/http/oauth2/filter.h index 0a0ce94860c7..f5e197649802 100644 --- a/source/extensions/filters/http/oauth2/filter.h +++ b/source/extensions/filters/http/oauth2/filter.h @@ -85,6 +85,7 @@ class SDSSecretReader : public SecretReader { #define ALL_OAUTH_FILTER_STATS(COUNTER) \ COUNTER(oauth_unauthorized_rq) \ COUNTER(oauth_failure) \ + COUNTER(oauth_passthrough) \ COUNTER(oauth_success) /** diff --git a/test/extensions/filters/http/oauth2/BUILD b/test/extensions/filters/http/oauth2/BUILD index 10b5b41cfa2b..502da9820d89 100644 --- a/test/extensions/filters/http/oauth2/BUILD +++ b/test/extensions/filters/http/oauth2/BUILD @@ -50,6 +50,7 @@ envoy_extension_cc_test( "//test/integration:http_integration_lib", "//test/mocks/server:server_mocks", "//test/mocks/upstream:upstream_mocks", + "//test/test_common:test_runtime_lib", "@envoy_api//envoy/extensions/filters/http/oauth2/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/filters/http/oauth2/filter_test.cc b/test/extensions/filters/http/oauth2/filter_test.cc index 143afef80ac2..d1c75795dce4 100644 --- a/test/extensions/filters/http/oauth2/filter_test.cc +++ b/test/extensions/filters/http/oauth2/filter_test.cc @@ -16,6 +16,7 @@ #include "test/mocks/http/mocks.h" #include "test/mocks/server/mocks.h" #include "test/mocks/upstream/mocks.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -573,11 +574,45 @@ TEST_F(OAuth2Test, OAuthOptionsRequestAndContinue) { {Http::Headers::get().Host.get(), "traffic.example.com"}, {Http::Headers::get().Path.get(), "/anypath"}, {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Options}, + {Http::CustomHeaders::get().Authorization.get(), "Bearer xyz-header-token"}}; + + Http::TestRequestHeaderMapImpl expected_headers{ + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Path.get(), "/anypath"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Options}, + {Http::CustomHeaders::get().Authorization.get(), "Bearer xyz-header-token"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(request_headers, expected_headers); + EXPECT_EQ(scope_.counterFromString("test.oauth_failure").value(), 0); + EXPECT_EQ(scope_.counterFromString("test.oauth_passthrough").value(), 1); + EXPECT_EQ(scope_.counterFromString("test.oauth_success").value(), 0); +} + +TEST_F(OAuth2Test, OAuthOptionsRequestAndContinue_oauth_header_passthrough_fix) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({ + {"envoy.reloadable_features.oauth_header_passthrough_fix", "false"}, + }); + Http::TestRequestHeaderMapImpl request_headers{ + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Path.get(), "/anypath"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Options}, + {Http::CustomHeaders::get().Authorization.get(), "Bearer xyz-header-token"}}; + + Http::TestRequestHeaderMapImpl expected_headers{ + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Path.get(), "/anypath"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Options}, }; EXPECT_CALL(*validator_, setParams(_, _)); EXPECT_CALL(*validator_, isValid()).WillOnce(Return(false)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(request_headers, expected_headers); + EXPECT_EQ(scope_.counterFromString("test.oauth_failure").value(), 0); + EXPECT_EQ(scope_.counterFromString("test.oauth_success").value(), 0); } // Validates the behavior of the cookie validator. From 1f176807e49558fd18a41a75ef4d81f8de11eff2 Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:41:38 -0400 Subject: [PATCH 045/112] docs: Fixing ECDS documentation (#23636) Signed-off-by: Yanjun Xiang --- .../extension/v3/config_discovery.proto | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/api/envoy/service/extension/v3/config_discovery.proto b/api/envoy/service/extension/v3/config_discovery.proto index f04986079d60..269b7e134169 100644 --- a/api/envoy/service/extension/v3/config_discovery.proto +++ b/api/envoy/service/extension/v3/config_discovery.proto @@ -18,6 +18,25 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Extension config discovery service (ECDS)] +// A service that supports dynamic configuration updates for a specific filter. +// Currently, ECDS is supported for HTTP filters and Listener filters. Please check +// :ref:`Extension Config Discovery Service (ECDS) API `. +// The overall extension config discovery service works as follows: +// +// 1. A filter (:ref:`Listener ` +// or :ref:`HTTP `) +// contains a :ref:`config_discovery ` configuration. This configuration +// includes a :ref:`config_source `, +// from which the filter configuration will be fetched. +// 2. The client then registers for a resource using the filter name as the resource_name. +// 3. The xDS server sends back the filter's configuration. +// 4. The client stores the configuration that will be used in the next instantiation of the filter chain, +// i.e., for the next requests. Whenever an updated filter configuration arrives, it will be taken into +// account in the following instantiation of the filter chain. +// +// Note: Filters that are configured using ECDS are warmed. For more details see +// :ref:`ExtensionConfigSource `. + // Return extension configurations. service ExtensionConfigDiscoveryService { option (envoy.annotations.resource).type = "envoy.config.core.v3.TypedExtensionConfig"; From f9daa53c352784043fbddc2f2d890c6419e5a361 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Mon, 31 Oct 2022 21:05:00 +0700 Subject: [PATCH 046/112] ext_authz: Remove unused test method (#23726) Signed-off-by: Dhi Aurrahman --- .../filters/http/ext_authz/ext_authz_integration_test.cc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc index 65752924572b..a3a9299bb62b 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc @@ -453,13 +453,6 @@ class ExtAuthzHttpIntegrationTest : public HttpIntegrationTest, addFakeUpstream(Http::CodecType::HTTP1); } - // By default, HTTP Service uses case sensitive string matcher. - void disableCaseSensitiveStringMatcher() { - config_helper_.addRuntimeOverride( - "envoy.reloadable_features.ext_authz_http_service_enable_case_sensitive_string_matcher", - "false"); - } - void initiateClientConnection() { auto conn = makeClientConnection(lookupPort("http")); codec_client_ = makeHttpConnection(std::move(conn)); From ceb45a32da8c9886c0eb87f8723068b6cd96c5f8 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Mon, 31 Oct 2022 21:44:30 +0700 Subject: [PATCH 047/112] tools, github: Avoid rate-limited calls when calling GitHub API (#23747) Signed-off-by: Dhi Aurrahman --- tools/github/write_current_source_version.py | 25 ++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tools/github/write_current_source_version.py b/tools/github/write_current_source_version.py index f183e577d3a0..9daccac70730 100644 --- a/tools/github/write_current_source_version.py +++ b/tools/github/write_current_source_version.py @@ -10,6 +10,7 @@ import argparse import json +import os import pathlib import sys import urllib.request @@ -21,6 +22,14 @@ dest="skip_error_in_git", help="Skip returning error on exit when the current directory is a git repository.", action="store_true") + parser.add_argument( + "--github_api_token_env_name", + dest="github_api_token_env_name", + help="The system environment variable name that holds GitHub API token. " + "This is advisable to provide this to avoid rate-limited calls.", + type=str, + action="store", + default="GITHUB_TOKEN") args = parser.parse_args() # Simple check if a .git directory exists. When we are in a Git repo, we should rely on git. @@ -35,8 +44,11 @@ sys.exit(0) sys.exit(1) + # Get the project root directory (../../..). + project_root_dir = pathlib.Path(__file__).parent.parent.parent + # Check if we have VERSION.txt available - current_version_file = pathlib.Path("VERSION.txt") + current_version_file = project_root_dir.joinpath("VERSION.txt") if not current_version_file.exists(): print( "Failed to read VERSION.txt. " @@ -54,9 +66,14 @@ sys.exit(1) # Fetch the current version commit information from GitHub. - with urllib.request.urlopen("https://api.github.com/repos/envoyproxy/envoy/commits/v" - + current_version) as response: + commit_info_request = urllib.request.Request( + "https://api.github.com/repos/envoyproxy/envoy/commits/v" + current_version) + github_token = os.environ.get(args.github_api_token_env_name) + if github_token != None: + # To avoid rate-limited API calls. + commit_info_request.add_header("Authorization", f"Bearer {github_token}") + with urllib.request.urlopen(commit_info_request) as response: commit_info = json.loads(response.read()) - source_version_file = pathlib.Path("SOURCE_VERSION") + source_version_file = project_root_dir.joinpath("SOURCE_VERSION") # Write the extracted current version commit hash "sha" to SOURCE_VERSION. source_version_file.write_text(commit_info["sha"]) From fd12444788cbb65557937ac46a46a38d139173ec Mon Sep 17 00:00:00 2001 From: Asher Yermiyahu Date: Mon, 31 Oct 2022 17:00:20 +0200 Subject: [PATCH 048/112] Fix premature termination of transaction client (#23710) Signed-off-by: asheryer --- .../network/redis_proxy/proxy_filter.cc | 2 +- .../redis_proxy_integration_test.cc | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/network/redis_proxy/proxy_filter.cc b/source/extensions/filters/network/redis_proxy/proxy_filter.cc index 775f9f8916be..51204c66a0ed 100644 --- a/source/extensions/filters/network/redis_proxy/proxy_filter.cc +++ b/source/extensions/filters/network/redis_proxy/proxy_filter.cc @@ -189,7 +189,7 @@ void ProxyFilter::onResponse(PendingRequest& request, Common::Redis::RespValuePt } // Check if there is an active transaction that needs to be closed. - if (transaction_.should_close_) { + if (transaction_.should_close_ && pending_requests_.empty()) { transaction_.close(); } } diff --git a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc index cdc08fdee891..9c9446225edc 100644 --- a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc +++ b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc @@ -1317,6 +1317,28 @@ TEST_P(RedisProxyWithCommandStatsIntegrationTest, SendMultiBeforeCommandInTransa redis_client->close(); } +// This test verifies that a transaction can be pipelined. +TEST_P(RedisProxyWithCommandStatsIntegrationTest, PipelinedTransactionTest) { + initialize(); + + std::array fake_upstream_connection; + std::string transaction_commands = + makeBulkStringArray({"MULTI"}) + makeBulkStringArray({"set", "foo", "bar"}) + + makeBulkStringArray({"get", "foo"}) + makeBulkStringArray({"exec"}); + const std::string& response = "+OK\r\n+QUEUED\r\n+QUEUED\r\n*2\r\n+OK\r\n$3\r\nbar\r\n"; + IntegrationTcpClientPtr redis_client = makeTcpConnection(lookupPort("redis_proxy")); + ASSERT_TRUE(redis_client->write(transaction_commands)); + + expectUpstreamRequestResponse(fake_upstreams_[0], transaction_commands, response, + fake_upstream_connection[0]); + + redis_client->waitForData(response); + EXPECT_EQ(response, redis_client->data()); + + EXPECT_TRUE(fake_upstream_connection[0]->close()); + redis_client->close(); +} + // TODO: Add full transaction test. } // namespace From c854b59e2851f88f813b0b0188253f9671ae8c2b Mon Sep 17 00:00:00 2001 From: Bootsie Heffernan Date: Mon, 31 Oct 2022 11:07:01 -0400 Subject: [PATCH 049/112] Header2 (#23723) Commit Message: Adding a header for grpc-message-type to CustomHeaderValues. Additional Description: N/a Risk Level: low Testing: unit Docs Changes: N/a Release Notes: N/a Platform Specific Features: N/a Signed-off-by: Bootsie Heffernan --- source/common/http/headers.h | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/http/headers.h b/source/common/http/headers.h index da764adefd58..4755c7832da0 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -74,6 +74,7 @@ class CustomHeaderValues { const LowerCaseString Expires{"expires"}; const LowerCaseString GrpcAcceptEncoding{"grpc-accept-encoding"}; const LowerCaseString GrpcEncoding{"grpc-encoding"}; + const LowerCaseString GrpcMessageEncoding{"grpc-message-encoding"}; const LowerCaseString IfMatch{"if-match"}; const LowerCaseString IfNoneMatch{"if-none-match"}; const LowerCaseString IfModifiedSince{"if-modified-since"}; From 362e415045e1c8b7391fcb1a21f8c6003de3e8e0 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Mon, 31 Oct 2022 22:47:33 +0700 Subject: [PATCH 050/112] build, macOS: Allow compiling v8 on macOS 10.15 to 13.0 (#23707) Signed-off-by: Dhi Aurrahman --- bazel/v8.patch | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/bazel/v8.patch b/bazel/v8.patch index 0cd409830f75..2d4a43c08f9a 100644 --- a/bazel/v8.patch +++ b/bazel/v8.patch @@ -1,7 +1,8 @@ # 1. Use already imported python dependencies # 2. Disable pointer compression (limits the maximum number of WasmVMs). -# 3. Add support for --define=no_debug_info=1. -# 4. Don't expose Wasm C API (only Wasm C++ API). +# 3. Add support for --define=no_debug_info=1. +# 4. Allow compiling v8 on macOS 10.15 to 13.0. TODO(dio): Will remove this patch when https://bugs.chromium.org/p/v8/issues/detail?id=13428 is fixed. +# 5. Don't expose Wasm C API (only Wasm C++ API). diff --git a/BUILD.bazel b/BUILD.bazel index 4e89f90e7e..ced403d5aa 100644 @@ -30,7 +31,7 @@ index 4e89f90e7e..3fcb38b3f3 100644 # Default setting for v8_enable_pointer_compression. diff --git a/bazel/defs.bzl b/bazel/defs.bzl -index e957c0fad3..eee285ab60 100644 +index e957c0fad3..5870a914aa 100644 --- a/bazel/defs.bzl +++ b/bazel/defs.bzl @@ -116,6 +116,7 @@ def _default_args(): @@ -41,7 +42,7 @@ index e957c0fad3..eee285ab60 100644 "-std=c++17", ], "@v8//bazel/config:is_gcc": [ -@@ -151,6 +152,11 @@ def _default_args(): +@@ -151,6 +152,23 @@ def _default_args(): "-fno-integrated-as", ], "//conditions:default": [], @@ -49,6 +50,18 @@ index e957c0fad3..eee285ab60 100644 + "@envoy//bazel:no_debug_info": [ + "-g0", + ], ++ "//conditions:default": [], ++ }) + select({ ++ "@v8//bazel/config:is_macos": [ ++ # The clang available on macOS catalina has a warning that isn't clean on v8 code. ++ "-Wno-range-loop-analysis", ++ ++ # To supress warning on deprecated declaration on v8 code. For example: ++ # external/v8/src/base/platform/platform-darwin.cc:56:22: 'getsectdatafromheader_64' ++ # is deprecated: first deprecated in macOS 13.0. ++ # https://bugs.chromium.org/p/v8/issues/detail?id=13428. ++ "-Wno-deprecated-declarations", ++ ], + "//conditions:default": [], }), includes = ["include"], From e3dee1b2810422a74cb4eca5175f016bd7692745 Mon Sep 17 00:00:00 2001 From: Loong Dai Date: Tue, 1 Nov 2022 00:15:57 +0800 Subject: [PATCH 051/112] remove envoy_reloadable_features_http_100_continue_case_insensitive (#23682) * envoy_reloadable_features_http_100_continue_case_insensitive deprecation Signed-off-by: Loong --- changelogs/current.yaml | 1 + source/common/http/conn_manager_impl.cc | 16 +++------- source/common/runtime/runtime_features.cc | 1 - test/integration/integration_test.cc | 36 ----------------------- 4 files changed, 5 insertions(+), 49 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 611324b94cb5..9e7f4441a4c3 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -52,6 +52,7 @@ removed_config_or_runtime: removed ``envoy.reloadable_features.http1_lazy_read_disable`` and legacy code paths. - area: http change: | + removed ``envoy.reloadable_features.http_100_continue_case_insensitive`` and legacy code paths. removed ``envoy.reloadable_features.override_request_timeout_by_gateway_timeout`` and legacy code paths. new_features: diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index da099cd82d68..565685ef26b2 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -948,19 +948,11 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapPtr&& he return; } - // This lambda should be erased when - // `envoy.reloadable_features.http_100_continue_case_insensitive` is removed. - auto is100Continue = [](absl::string_view request_expect) { - return request_expect == Headers::get().ExpectValues._100Continue || - (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.http_100_continue_case_insensitive") && - // The Expect field-value is case-insensitive. - // https://tools.ietf.org/html/rfc7231#section-5.1.1 - absl::EqualsIgnoreCase(request_expect, Headers::get().ExpectValues._100Continue)); - }; - if (!connection_manager_.config_.proxy100Continue() && request_headers_->Expect() && - is100Continue(request_headers_->Expect()->value().getStringView())) { + // The Expect field-value is case-insensitive. + // https://tools.ietf.org/html/rfc7231#section-5.1.1 + absl::EqualsIgnoreCase((request_headers_->Expect()->value().getStringView()), + Headers::get().ExpectValues._100Continue)) { // Note in the case Envoy is handling 100-Continue complexity, it skips the filter chain // and sends the 100-Continue directly to the encoder. chargeStats(continueHeader()); diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index a70095769234..b38867e5f29d 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -47,7 +47,6 @@ RUNTIME_GUARD(envoy_reloadable_features_fix_hash_key); RUNTIME_GUARD(envoy_reloadable_features_get_route_config_factory_by_type); RUNTIME_GUARD(envoy_reloadable_features_http2_delay_keepalive_timeout); RUNTIME_GUARD(envoy_reloadable_features_http3_sends_early_data); -RUNTIME_GUARD(envoy_reloadable_features_http_100_continue_case_insensitive); RUNTIME_GUARD(envoy_reloadable_features_http_reject_path_with_fragment); RUNTIME_GUARD(envoy_reloadable_features_http_response_half_close); RUNTIME_GUARD(envoy_reloadable_features_http_skip_adding_content_length_to_upgrade); diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index aa52f855f91a..c18c30ec7cf1 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -496,42 +496,6 @@ TEST_P(IntegrationTest, EnvoyProxyingLate1xxWithEncoderFilter) { testEnvoyProxying1xx(false, true); } -// When the runtime feature `http_100_continue_case_insensitive` is disabled, the "100-Continue" -// (upper case C) is not counted as "100-continue". As a consequence, the response does not contain -// the `100` status code as if Envoy does not see `expect` header. -TEST_P(IntegrationTest, RuntimeFeature100ContinueCaseInsensitiveDisabled) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.http_100_continue_case_insensitive", - "false"); - - config_helper_.addConfigModifier( - [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& - hcm) -> void { hcm.set_proxy_100_continue(false); }); - initialize(); - - codec_client_ = makeHttpConnection(lookupPort("http")); - auto encoder_decoder = - codec_client_->startRequest(Http::TestRequestHeaderMapImpl{{":method", "GET"}, - {":path", "/dynamo/url"}, - {":scheme", "http"}, - {":authority", "sni.lyft.com"}, - {"expect", "100-Continue"}}); - request_encoder_ = &encoder_decoder.first; - auto response = std::move(encoder_decoder.second); - - // Send all of the request data and wait for it to be received upstream. - ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); - ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); - codec_client_->sendData(*request_encoder_, 10, true); - ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); - upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(response->complete()); - - // The response contains the status code 200 but does not contain the status code 100. - EXPECT_EQ(nullptr, response->informationalHeaders()); - EXPECT_EQ("200", response->headers().getStatusValue()); -} - // Regression test for https://github.com/envoyproxy/envoy/issues/10923. TEST_P(IntegrationTest, EnvoyProxying1xxWithDecodeDataPause) { config_helper_.prependFilter(R"EOF( From a2b595da4fd48c7c6fa399637f5001d0c0f0f6a5 Mon Sep 17 00:00:00 2001 From: Jonatan Vela <113522150+ergonjona@users.noreply.github.com> Date: Mon, 31 Oct 2022 17:25:23 +0100 Subject: [PATCH 052/112] Explicitly disable libpsl for curl (#23748) Risk: Low Testing: test/dependencies/curl_test.cc tests that libpsl is disabled On my workstation (running Arch Linux) I had a linker error: ld.lld: error: undefined symbol: psl_is_cookie_domain_acceptable >>> referenced by cookie.c >>> cookie.c.o:(Curl_cookie_add) in archive bazel-out/k8-opt/bin/external/envoy/bazel/foreign_cc/curl/lib/libcurl.a ... In test/dependencies/curl_test.cc I see that it is expected that curl does not use libpsl. Apparently did curl somehow compile with libpsl support but failed to link against it since it is not included in Envoy. Explicitly disabling libpsl fixed my problem. Signed-off-by: Jonatan Vela --- bazel/foreign_cc/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index 6d7fd1b2045e..d885daa9702d 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -264,6 +264,7 @@ envoy_cmake( "BUILD_SHARED_LIBS": "off", "CURL_HIDDEN_SYMBOLS": "off", "CURL_USE_LIBSSH2": "off", + "CURL_USE_LIBPSL": "off", "CURL_BROTLI": "off", "CURL_USE_GSSAPI": "off", "HTTP_ONLY": "on", From 2e62a653ccef29e28e06842cbf34fb5f510f4a0c Mon Sep 17 00:00:00 2001 From: Xie Zhihao Date: Tue, 1 Nov 2022 01:32:02 +0800 Subject: [PATCH 053/112] test: wait for response in request timeout integration test (#23746) Signed-off-by: Xie Zhihao --- test/integration/idle_timeout_integration_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/idle_timeout_integration_test.cc b/test/integration/idle_timeout_integration_test.cc index 80840ad2b99c..19f2573641e3 100644 --- a/test/integration/idle_timeout_integration_test.cc +++ b/test/integration/idle_timeout_integration_test.cc @@ -473,7 +473,7 @@ TEST_P(IdleTimeoutIntegrationTest, RequestTimeoutTriggersOnRawIncompleteRequestW initialize(); std::string raw_response; - sendRawHttpAndWaitForResponse(lookupPort("http"), "GET / HTTP/1.1", &raw_response, true); + sendRawHttpAndWaitForResponse(lookupPort("http"), "GET / HTTP/1.1", &raw_response); EXPECT_THAT(raw_response, testing::HasSubstr("request timeout")); } @@ -486,7 +486,7 @@ TEST_P(IdleTimeoutIntegrationTest, RequestTimeoutDoesNotTriggerOnRawCompleteRequ initialize(); std::string raw_response; - sendRawHttpAndWaitForResponse(lookupPort("http"), "GET / HTTP/1.1\r\n\r\n", &raw_response, true); + sendRawHttpAndWaitForResponse(lookupPort("http"), "GET / HTTP/1.1\r\n\r\n", &raw_response); EXPECT_THAT(raw_response, testing::Not(testing::HasSubstr("request timeout"))); } From e082dbb164643a80eef98526f46cead19f42e9cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=89=A7=E5=BF=83?= Date: Tue, 1 Nov 2022 01:38:55 +0800 Subject: [PATCH 054/112] Chore: correct `write_current_source_version.py` link (#23749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit correct `write_current_source_version.py` link Signed-off-by: 牧心 --- bazel/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bazel/README.md b/bazel/README.md index 97ee7308b365..4e8322155508 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -38,10 +38,10 @@ To build Envoy from a release tarball, you can download a release tarball from A Given all required [Envoy dependencies](https://www.envoyproxy.io/docs/envoy/latest/start/building#requirements) are installed, the following steps should be followed: 1. Download and extract source code of a release tarball from the Releases page. For example: https://github.com/envoyproxy/envoy/releases/tag/v1.24.0. -1. `python3 tools/github/tools/github/write_current_source_version.py` from the repository root. +1. `python3 tools/github/write_current_source_version.py` from the repository root. 1. `bazel build -c opt envoy` from the repository root. -> Note: If the the `write_current_source_version.py` script is missing from the extracted source code directory, you can download it from [here](https://raw.githubusercontent.com/envoyproxy/envoy/tree/main/tools/github/write_current_source_version.py). +> Note: If the the `write_current_source_version.py` script is missing from the extracted source code directory, you can download it from [here](https://raw.githubusercontent.com/envoyproxy/envoy/main/tools/github/write_current_source_version.py). > This script is used to generate SOURCE_VERSION that is required by [`bazel/get_workspace_status`](./get_workspace_status) to "stamp" the binary in a non-git directory. ## Quick start Bazel build for developers From 22d0ffc854ea7932bfb692a50fd9d75b412163c8 Mon Sep 17 00:00:00 2001 From: Tony Han Date: Tue, 1 Nov 2022 01:47:46 +0800 Subject: [PATCH 055/112] router: set stream_info_'s request headers of upstream request (#23684) Signed-off-by: Bing Han --- source/common/router/upstream_request.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index 6f86a3c7d8c3..178e06a29f59 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -667,6 +667,8 @@ void UpstreamRequest::onPoolReady(std::unique_ptr&& upstream, parent_.callbacks()->activeSpan().injectContext(*parent_.downstreamHeaders(), host); } + stream_info_.setRequestHeaders(*parent_.downstreamHeaders()); + for (auto* callback : upstream_callbacks_) { callback->onUpstreamConnectionEstablished(); return; From 66e5e6f9ee201a4ebdb7c33cd83d30a72ca357d4 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 31 Oct 2022 12:06:16 -0700 Subject: [PATCH 056/112] bazel: avoid type-limits warning on arm64 Linux (#23757) Related: https://github.com/envoyproxy/envoy/issues/23580#issuecomment-1296488433 Signed-off-by: Keith Smiley --- .bazelrc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.bazelrc b/.bazelrc index 240236de515f..eea01acb6217 100644 --- a/.bazelrc +++ b/.bazelrc @@ -45,6 +45,10 @@ build:linux --features=per_object_debug_info build:linux --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a build:linux --action_env=BAZEL_LINKOPTS=-lm +# TODO(keith): remove once https://github.com/DataDog/dd-opentracing-cpp/pull/252 is integrated +# this avoids warnings/errors on arm64 Linux builds +build:linux --per_file_copt=external/com_github_datadog_dd_opentracing_cpp/.*.cpp@-Wno-type-limits + # We already have absl in the build, define absl=1 to tell googletest to use absl for backtrace. build --define absl=1 From 6b7c95d16667b757110df0f7dd234b4be6d914bf Mon Sep 17 00:00:00 2001 From: Sean Maloney Date: Mon, 31 Oct 2022 15:34:59 -0400 Subject: [PATCH 057/112] Test validation for wildcard match in middle of path (#23752) Test validation for wildcard match in middle of path Additional Description: Pattern matching does not support wildcard matching in the middle of a path. This test checks that it is not allowed in config. Signed-off-by: silverstar195 --- test/common/router/config_impl_test.cc | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 164d9029cbce..0740ac575667 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -9314,6 +9314,35 @@ TEST_F(RouteMatcherTest, PatternMatchInvalidVariableName) { EnvoyException, "path_match_policy.path_template /rest/{on==e}/{two} is invalid"); } +TEST_F(RouteMatcherTest, PatternMatchInvalidPlacedWildcard) { + const std::string yaml = R"EOF( +virtual_hosts: + - name: path_pattern + domains: ["*"] + routes: + - match: + path_match_policy: + name: envoy.path.match.uri_template.pattern_template_match_predicate + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig + path_template: "/rest/{middlewildcard=**}/{two}" + case_sensitive: false + route: + cluster: "path-pattern-cluster-one" + path_rewrite_policy: + name: envoy.path.rewrite.uri_template.pattern_template_rewrite_predicate + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig + path_template_rewrite: "/rest/{middlewildcard=**}/{two}" + )EOF"; + NiceMock stream_info; + factory_context_.cluster_manager_.initializeClusters({"path-pattern-cluster-one"}, {}); + + EXPECT_THROW_WITH_MESSAGE( + TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true), + EnvoyException, "path_match_policy.path_template /rest/{middlewildcard=**}/{two} is invalid"); +} + TEST_F(RouteMatcherTest, PatternMatchWildcardUnnamedVariable) { const std::string yaml = R"EOF( virtual_hosts: From ec07bf61135bb73839091ffc269c00cc922ee28e Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 31 Oct 2022 17:25:32 -0700 Subject: [PATCH 058/112] bazel: workaround invalid nonnull warning in v8 (#23765) Since gcc 12 it seems there are some false positives around this warning. Upstream gcc reports: - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107252 - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100366 - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106199 We can't use `--per_file_copt` in this case because we build this in the host configuration https://github.com/bazelbuild/bazel/issues/12406 Upstream v8 report: https://bugs.chromium.org/p/v8/issues/detail?id=13441 Fixes https://github.com/envoyproxy/envoy/issues/23692 Signed-off-by: Keith Smiley --- bazel/v8.patch | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bazel/v8.patch b/bazel/v8.patch index 2d4a43c08f9a..78a8411ad46e 100644 --- a/bazel/v8.patch +++ b/bazel/v8.patch @@ -31,7 +31,7 @@ index 4e89f90e7e..3fcb38b3f3 100644 # Default setting for v8_enable_pointer_compression. diff --git a/bazel/defs.bzl b/bazel/defs.bzl -index e957c0fad3..5870a914aa 100644 +index e957c0fad3..a6de50e6ab 100644 --- a/bazel/defs.bzl +++ b/bazel/defs.bzl @@ -116,6 +116,7 @@ def _default_args(): @@ -42,7 +42,15 @@ index e957c0fad3..5870a914aa 100644 "-std=c++17", ], "@v8//bazel/config:is_gcc": [ -@@ -151,6 +152,23 @@ def _default_args(): +@@ -131,6 +132,7 @@ def _default_args(): + "-Wno-redundant-move", + "-Wno-return-type", + "-Wno-stringop-overflow", ++ "-Wno-nonnull", + # Use GNU dialect, because GCC doesn't allow using + # ##__VA_ARGS__ when in standards-conforming mode. + "-std=gnu++17", +@@ -151,6 +153,23 @@ def _default_args(): "-fno-integrated-as", ], "//conditions:default": [], From ef687b02506d4b5acc6a7e1a6a1f379f76cd10af Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 31 Oct 2022 21:42:54 -0700 Subject: [PATCH 059/112] bazel: remove genrule_repository (#23768) This rule seems to have been used a lot more in the past, likely pre rules_foreign_cc. At this point it was only used for boringssl_fips, and we can easily replace that use with a normal genrule that just calls the same script. The script had to be updated slightly to remove the few dependencies on bazel specifics. Signed-off-by: Keith Smiley --- bazel/EXTERNAL_DEPS.md | 26 ---- bazel/external/BUILD | 2 + bazel/external/boringssl_fips.BUILD | 5 +- bazel/external/boringssl_fips.genrule_cmd | 53 +++++---- bazel/genrule_repository.bzl | 138 ---------------------- bazel/repositories.bzl | 12 +- 6 files changed, 32 insertions(+), 204 deletions(-) mode change 100644 => 100755 bazel/external/boringssl_fips.genrule_cmd delete mode 100644 bazel/genrule_repository.bzl diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md index f5ea35a7f1b7..b7919b5b06c8 100644 --- a/bazel/EXTERNAL_DEPS.md +++ b/bazel/EXTERNAL_DEPS.md @@ -31,32 +31,6 @@ This is the preferred style of adding dependencies that use CMake for their buil `external_deps` attribute. 4. `bazel test //test/...` - -## genrule repository - -This is the newer style of adding dependencies with no upstream Bazel configs. -It wraps the dependency's native build tooling in a Bazel-aware shell script, -installing to a Bazel-managed prefix. - -The shell script is executed by Bash, with a few Bazel-specific extensions. -See the [Bazel docs for "genrule"](https://docs.bazel.build/versions/master/be/general.html#genrule) -for details on Bazel's shell extensions. - -1. Add a BUILD file in [`bazel/external/`](external/), using a `genrule` target - to build the dependency. Please do not add BUILD logic that replaces the - dependency's upstream build tooling. -2. Define a new Bazel repository in [`bazel/repositories.bzl`](repositories.bzl), - in the `envoy_dependencies()` function. The repository may use `genrule_repository` - from [`bazel/genrule_repository.bzl`](genrule_repository.bzl) to place large - genrule shell commands into a separate file. -3. Reference your new external dependency in some `envoy_cc_library` via Y in the - `external_deps` attribute. -4. `bazel test //test/...` - -Dependencies between external libraries can use the standard Bazel dependency -resolution logic, using the `$(location)` shell extension to resolve paths -to binaries, libraries, headers, etc. - # Adding external dependencies to Envoy (Python) Python dependencies should be added via `pip` and `rules_python`. The process diff --git a/bazel/external/BUILD b/bazel/external/BUILD index add40009318d..62a6ca994d26 100644 --- a/bazel/external/BUILD +++ b/bazel/external/BUILD @@ -1,5 +1,7 @@ licenses(["notice"]) # Apache 2 +exports_files(["boringssl_fips.genrule_cmd"]) + # Use a wrapper cc_library with an empty source source file to force # compilation of other cc_library targets that only list *.a sources. cc_library( diff --git a/bazel/external/boringssl_fips.BUILD b/bazel/external/boringssl_fips.BUILD index 94fca2ac4c89..1af9f34b1f02 100644 --- a/bazel/external/boringssl_fips.BUILD +++ b/bazel/external/boringssl_fips.BUILD @@ -1,5 +1,3 @@ -load(":genrule_cmd.bzl", "genrule_cmd") - licenses(["notice"]) # Apache 2 cc_library( @@ -31,5 +29,6 @@ genrule( "crypto/libcrypto.a", "ssl/libssl.a", ], - cmd = genrule_cmd("@envoy//bazel/external:boringssl_fips.genrule_cmd"), + cmd = "$(location {}) $(location crypto/libcrypto.a) $(location ssl/libssl.a)".format("@envoy//bazel/external:boringssl_fips.genrule_cmd"), + exec_tools = ["@envoy//bazel/external:boringssl_fips.genrule_cmd"], ) diff --git a/bazel/external/boringssl_fips.genrule_cmd b/bazel/external/boringssl_fips.genrule_cmd old mode 100644 new mode 100755 index 25455c91e564..b4036e9bb9f0 --- a/bazel/external/boringssl_fips.genrule_cmd +++ b/bazel/external/boringssl_fips.genrule_cmd @@ -12,31 +12,32 @@ if [[ `uname` != "Linux" || `uname -m` != "x86_64" ]]; then fi # Bazel magic. -ROOT=$$(dirname $(rootpath boringssl/BUILDING.md))/.. -pushd $$ROOT +# ROOT=$(dirname $(rootpath boringssl/BUILDING.md))/.. +ROOT=./external/boringssl_fips +pushd "$ROOT" # Build tools requirements: # - Clang compiler version 7.0.1 (https://releases.llvm.org/download.html) # - Go programming language version 1.12.7 (https://golang.org/dl/) # - Ninja build system version 1.9.0 (https://github.com/ninja-build/ninja/releases) -# Override $$PATH for build tools, to avoid picking up anything else. -export PATH="$$(dirname `which cmake`):/usr/bin:/bin" +# Override $PATH for build tools, to avoid picking up anything else. +export PATH="$(dirname `which cmake`):/usr/bin:/bin" # Clang 7.0.1 VERSION=7.0.1 SHA256=02ad925add5b2b934d64c3dd5cbd1b2002258059f7d962993ba7f16524c3089c PLATFORM="x86_64-linux-gnu-ubuntu-16.04" -curl -sLO https://releases.llvm.org/"$$VERSION"/clang+llvm-"$$VERSION"-"$$PLATFORM".tar.xz \ - && echo "$$SHA256" clang+llvm-"$$VERSION"-"$$PLATFORM".tar.xz | sha256sum --check -tar xf clang+llvm-"$$VERSION"-"$$PLATFORM".tar.xz +curl -sLO https://releases.llvm.org/"$VERSION"/clang+llvm-"$VERSION"-"$PLATFORM".tar.xz \ + && echo "$SHA256" clang+llvm-"$VERSION"-"$PLATFORM".tar.xz | sha256sum --check +tar xf clang+llvm-"$VERSION"-"$PLATFORM".tar.xz -export HOME="$$PWD" -printf "set(CMAKE_C_COMPILER \"clang\")\nset(CMAKE_CXX_COMPILER \"clang++\")\n" > $${HOME}/toolchain -export PATH="$$PWD/clang+llvm-$$VERSION-$$PLATFORM/bin:$$PATH" +export HOME="$PWD" +printf "set(CMAKE_C_COMPILER \"clang\")\nset(CMAKE_CXX_COMPILER \"clang++\")\n" > ${HOME}/toolchain +export PATH="$PWD/clang+llvm-$VERSION-$PLATFORM/bin:$PATH" -if [[ `clang --version | head -1 | awk '{print $$3}'` != "$$VERSION" ]]; then +if [[ `clang --version | head -1 | awk '{print $3}'` != "$VERSION" ]]; then echo "ERROR: Clang version doesn't match." exit 1 fi @@ -46,15 +47,15 @@ VERSION=1.12.7 SHA256=66d83bfb5a9ede000e33c6579a91a29e6b101829ad41fffb5c5bb6c900e109d9 PLATFORM="linux-amd64" -curl -sLO https://dl.google.com/go/go"$$VERSION"."$$PLATFORM".tar.gz \ - && echo "$$SHA256" go"$$VERSION"."$$PLATFORM".tar.gz | sha256sum --check -tar xf go"$$VERSION"."$$PLATFORM".tar.gz +curl -sLO https://dl.google.com/go/go"$VERSION"."$PLATFORM".tar.gz \ + && echo "$SHA256" go"$VERSION"."$PLATFORM".tar.gz | sha256sum --check +tar xf go"$VERSION"."$PLATFORM".tar.gz -export GOPATH="$$PWD/gopath" -export GOROOT="$$PWD/go" -export PATH="$$GOPATH/bin:$$GOROOT/bin:$$PATH" +export GOPATH="$PWD/gopath" +export GOROOT="$PWD/go" +export PATH="$GOPATH/bin:$GOROOT/bin:$PATH" -if [[ `go version | awk '{print $$3}'` != "go$$VERSION" ]]; then +if [[ `go version | awk '{print $3}'` != "go$VERSION" ]]; then echo "ERROR: Go version doesn't match." exit 1 fi @@ -64,13 +65,13 @@ VERSION=1.9.0 SHA256=1b1235f2b0b4df55ac6d80bbe681ea3639c9d2c505c7ff2159a3daf63d196305 PLATFORM="linux" -curl -sLO https://github.com/ninja-build/ninja/releases/download/v"$$VERSION"/ninja-"$$PLATFORM".zip \ - && echo "$$SHA256" ninja-"$$PLATFORM".zip | sha256sum --check -unzip -o ninja-"$$PLATFORM".zip +curl -sLO https://github.com/ninja-build/ninja/releases/download/v"$VERSION"/ninja-"$PLATFORM".zip \ + && echo "$SHA256" ninja-"$PLATFORM".zip | sha256sum --check +unzip -o ninja-"$PLATFORM".zip -export PATH="$$PWD:$$PATH" +export PATH="$PWD:$PATH" -if [[ `ninja --version` != "$$VERSION" ]]; then +if [[ `ninja --version` != "$VERSION" ]]; then echo "ERROR: Ninja version doesn't match." exit 1 fi @@ -80,7 +81,7 @@ rm -rf boringssl/build # Build BoringSSL. cd boringssl -mkdir build && cd build && cmake -GNinja -DCMAKE_TOOLCHAIN_FILE=$${HOME}/toolchain -DFIPS=1 -DCMAKE_BUILD_TYPE=Release .. +mkdir build && cd build && cmake -GNinja -DCMAKE_TOOLCHAIN_FILE=${HOME}/toolchain -DFIPS=1 -DCMAKE_BUILD_TYPE=Release .. ninja ninja run_tests @@ -92,5 +93,5 @@ fi # Move compiled libraries to the expected destinations. popd -mv $$ROOT/boringssl/build/crypto/libcrypto.a $(execpath crypto/libcrypto.a) -mv $$ROOT/boringssl/build/ssl/libssl.a $(execpath ssl/libssl.a) +mv $ROOT/boringssl/build/crypto/libcrypto.a $1 +mv $ROOT/boringssl/build/ssl/libssl.a $2 diff --git a/bazel/genrule_repository.bzl b/bazel/genrule_repository.bzl deleted file mode 100644 index e263c43d4689..000000000000 --- a/bazel/genrule_repository.bzl +++ /dev/null @@ -1,138 +0,0 @@ -def _genrule_repository(ctx): - ctx.download_and_extract( - ctx.attr.urls, - "", # output - ctx.attr.sha256, - "", # type - ctx.attr.strip_prefix, - ) - for ii, patch in enumerate(ctx.attr.patches): - patch_input = "patch-input-%d.patch" % (ii,) - ctx.symlink(patch, patch_input) - patch_result = ctx.execute(["patch", "-p0", "--input", patch_input]) - if patch_result.return_code != 0: - fail("Failed to apply patch %r: %s, %s" % (patch, patch_result.stderr, patch_result.stdout)) - - genrule_cmd = ctx.read(ctx.attr.genrule_cmd_file) - ctx.file("WORKSPACE", "workspace(name=%r)" % (ctx.name,)) - ctx.delete("BUILD.bazel") - ctx.symlink(ctx.attr.build_file, "BUILD.bazel") - - # Inject the genrule_cmd content into a .bzl file that can be loaded - # from the repository BUILD file. We force the user to look up the - # command content "by label" so the inclusion source is obvious. - ctx.file("genrule_cmd.bzl", """ -_GENRULE_CMD = {%r: %r} -def genrule_cmd(label): - return _GENRULE_CMD[Label(label)] -""" % (ctx.attr.genrule_cmd_file, genrule_cmd)) - -genrule_repository = repository_rule( - attrs = { - "urls": attr.string_list( - mandatory = True, - allow_empty = False, - ), - "sha256": attr.string(), - "strip_prefix": attr.string(), - "patches": attr.label_list( - allow_files = [".patch"], - allow_empty = True, - ), - "genrule_cmd_file": attr.label( - mandatory = True, - allow_single_file = [".genrule_cmd"], - ), - "build_file": attr.label( - mandatory = True, - allow_single_file = [".BUILD"], - ), - }, - implementation = _genrule_repository, -) - -def _genrule_cc_deps(ctx): - outs = depset() - for dep in ctx.attr.deps: - outs = dep.cc.transitive_headers + dep.cc.libs + outs - return DefaultInfo(files = outs) - -genrule_cc_deps = rule( - attrs = { - "deps": attr.label_list( - providers = [], # CcStarlarkApiProvider - mandatory = True, - allow_empty = False, - ), - }, - implementation = _genrule_cc_deps, -) - -def _absolute_bin(path): - # If the binary path looks like it's relative to the current directory, - # transform it to be absolute by appending "${PWD}". - if "/" in path and not path.startswith("/"): - return '"${PWD}"/%r' % (path,) - return "%r" % (path,) - -def _genrule_environment(ctx): - lines = [] - - # Bazel uses the same command for C and C++ compilation. - c_compiler = ctx.var["CC"] - - # Bare minimum cflags to get included test binaries to link. - # - # See .bazelrc for the full set. - asan_flags = ["-fsanitize=address,undefined"] - tsan_flags = ["-fsanitize=thread"] - - # Older versions of GCC in Ubuntu, including GCC 5 used in CI images, - # incorrectly invoke the older `/usr/bin/ld` with gold-specific options when - # building with sanitizers enabled. Work around this by forcing use of gold - # in sanitize mode. - # - # This is not a great solution because it doesn't detect GCC when Bazel has - # wrapped it in an intermediate script, but it works well enough to keep CI - # running. - # - # https://stackoverflow.com/questions/37603238/fsanitize-not-using-gold-linker-in-gcc-6-1 - force_ld = [] - if "clang" in c_compiler: - force_ld = ["-fuse-ld=lld"] - elif "gcc" in c_compiler or "g++" in c_compiler: - force_ld = ["-fuse-ld=gold"] - - cc_flags = [] - ld_flags = [] - ld_libs = [] - if ctx.var.get("ENVOY_CONFIG_COVERAGE"): - ld_libs.append("-lgcov") - if ctx.var.get("ENVOY_CONFIG_ASAN"): - cc_flags += asan_flags - ld_flags += asan_flags - ld_flags += force_ld - if ctx.var.get("ENVOY_CONFIG_TSAN"): - cc_flags += tsan_flags - ld_flags += tsan_flags - ld_flags += force_ld - - lines.append("export CFLAGS=%r" % (" ".join(cc_flags),)) - lines.append("export LDFLAGS=%r" % (" ".join(ld_flags),)) - lines.append("export LIBS=%r" % (" ".join(ld_libs),)) - lines.append("export CC=%s" % (_absolute_bin(c_compiler),)) - lines.append("export CXX=%s" % (_absolute_bin(c_compiler),)) - - # Some Autoconf helper binaries leak, which makes ./configure think the - # system is unable to do anything. Turn off leak checking during part of - # the build. - lines.append("export ASAN_OPTIONS=detect_leaks=0") - - lines.append("") - out = ctx.actions.declare_file(ctx.attr.name + ".sh") - ctx.actions.write(out, "\n".join(lines)) - return DefaultInfo(files = depset([out])) - -genrule_environment = rule( - implementation = _genrule_environment, -) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 2bef75a72874..dcb8d10ecba3 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1,5 +1,4 @@ load(":dev_binding.bzl", "envoy_dev_binding") -load(":genrule_repository.bzl", "genrule_repository") load("@envoy_api//bazel:envoy_http_archive.bzl", "envoy_http_archive") load("@envoy_api//bazel:external_deps.bzl", "load_repository_locations") load(":repository_locations.bzl", "PROTOC_VERSIONS", "REPOSITORY_LOCATIONS_SPEC") @@ -33,14 +32,6 @@ def external_http_archive(name, **kwargs): **kwargs ) -# Use this macro to reference any genrule_repository sourced from bazel/repository_locations.bzl. -def external_genrule_repository(name, **kwargs): - location = REPOSITORY_LOCATIONS[name] - genrule_repository( - name = name, - **dict(location, **kwargs) - ) - def _default_envoy_build_config_impl(ctx): ctx.file("WORKSPACE", "") ctx.file("BUILD.bazel", "") @@ -271,9 +262,8 @@ def _boringssl(): ) def _boringssl_fips(): - external_genrule_repository( + external_http_archive( name = "boringssl_fips", - genrule_cmd_file = "@envoy//bazel/external:boringssl_fips.genrule_cmd", build_file = "@envoy//bazel/external:boringssl_fips.BUILD", patches = ["@envoy//bazel/external:boringssl_fips.patch"], ) From 36fc1ccc1de30b79594ca5ef9d8f0799a0f399a0 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Tue, 1 Nov 2022 18:23:51 +0700 Subject: [PATCH 060/112] tools, github: Pick authorization prefix based on provided token (#23766) Signed-off-by: Dhi Aurrahman --- tools/github/write_current_source_version.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/github/write_current_source_version.py b/tools/github/write_current_source_version.py index 9daccac70730..684245b98c6f 100644 --- a/tools/github/write_current_source_version.py +++ b/tools/github/write_current_source_version.py @@ -68,10 +68,12 @@ # Fetch the current version commit information from GitHub. commit_info_request = urllib.request.Request( "https://api.github.com/repos/envoyproxy/envoy/commits/v" + current_version) - github_token = os.environ.get(args.github_api_token_env_name) - if github_token != None: + if github_token := os.environ.get(args.github_api_token_env_name): + # Reference: https://github.com/octokit/auth-token.js/blob/902a172693d08de998250bf4d8acb1fdb22377a4/src/with-authorization-prefix.ts#L6-L12. + authorization_header_prefix = "bearer" if len(github_token.split(".")) == 3 else "token" # To avoid rate-limited API calls. - commit_info_request.add_header("Authorization", f"Bearer {github_token}") + commit_info_request.add_header( + "Authorization", f"{authorization_header_prefix} {github_token}") with urllib.request.urlopen(commit_info_request) as response: commit_info = json.loads(response.read()) source_version_file = project_root_dir.joinpath("SOURCE_VERSION") From 9a3b54136e8f22bc5a62c479036c61f54bd417f1 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 1 Nov 2022 12:58:36 +0000 Subject: [PATCH 061/112] repo: Yaml cleanups (#23654) Signed-off-by: Ryan Northey --- .yamllint | 13 +- .zuul.yaml | 2 +- api/buf.yaml | 28 ++-- changelogs/1.13.0.yaml | 2 +- changelogs/1.19.5.yaml | 1 - changelogs/1.20.4.yaml | 1 - changelogs/1.20.5.yaml | 1 - changelogs/1.20.6.yaml | 1 - changelogs/1.20.7.yaml | 1 - changelogs/1.21.2.yaml | 1 - changelogs/1.21.3.yaml | 1 - changelogs/1.21.4.yaml | 1 - changelogs/1.21.5.yaml | 1 - changelogs/1.22.1.yaml | 1 - changelogs/1.22.2.yaml | 1 - changelogs/1.22.3.yaml | 1 - changelogs/1.22.4.yaml | 1 - changelogs/1.22.5.yaml | 1 - changelogs/1.23.0.yaml | 1 - changelogs/1.23.1.yaml | 1 - changelogs/1.23.2.yaml | 1 - changelogs/1.24.0.yaml | 1 - changelogs/current.yaml | 55 ++++--- .../encapsulate_http_in_http2_connect.yaml | 6 +- configs/internal_listener_proxy.yaml | 6 +- configs/terminate_http1_connect.yaml | 6 +- configs/terminate_http_in_http2_connect.yaml | 6 +- configs/upstream-filters.yaml | 21 ++- distribution/distros.yaml | 1 - .../_include/admission-control-filter.yaml | 5 +- .../_include/header-to-metadata-filter.yaml | 8 +- ...local-rate-limit-global-configuration.yaml | 12 +- ...te-limit-route-specific-configuration.yaml | 16 +- .../local-rate-limit-with-descriptors.yaml | 8 +- .../_include/header-to-metadata-filter.yaml | 8 +- .../intro/_include/life-of-a-request.yaml | 1 - .../_include/listener_complicated.yaml | 2 +- .../matching/_include/listener_tls.yaml | 2 +- .../matching/_include/listener_vip.yaml | 2 +- examples/brotli/brotli-envoy.yaml | 4 +- examples/kafka/envoy.yaml | 4 +- examples/local_ratelimit/ratelimit-envoy.yaml | 16 +- .../locality-load-balancing/envoy-proxy.yaml | 74 +++++----- examples/wasm-cc/docker-compose.yaml | 2 +- examples/zstd/zstd-envoy.yaml | 4 +- source/extensions/extensions_metadata.yaml | 34 ++--- test/config/integration/server.yaml | 2 +- .../server_multiple_addresses.yaml | 2 +- test/config/integration/server_xds.lds.yaml | 1 - .../filters/http/ext_authz/ext_authz.yaml | 138 +++++++++--------- .../server/access_log_filter_bootstrap.yaml | 2 +- tools/code_format/config.yaml | 14 +- tools/dependency/cve.yaml | 1 - tools/extensions/extensions_schema.yaml | 1 - 54 files changed, 254 insertions(+), 273 deletions(-) diff --git a/.yamllint b/.yamllint index 05a553539fa2..01b057f6aa8e 100644 --- a/.yamllint +++ b/.yamllint @@ -1,9 +1,16 @@ extends: default rules: - line-length: - max: 200 - level: warning + document-start: false indentation: spaces: consistent indent-sequences: false + line-length: + max: 200 + level: warning + truthy: + allowed-values: + - "yes" + - "no" + - "true" + - "false" diff --git a/.zuul.yaml b/.zuul.yaml index e1e01e446e6b..1d02dbd13331 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -2,7 +2,7 @@ name: envoyproxy/envoy check: jobs: - - envoy-build-arm64 + - envoy-build-arm64 - job: name: envoy-build-arm64 diff --git a/api/buf.yaml b/api/buf.yaml index f5373484ed71..ef4031f27f28 100644 --- a/api/buf.yaml +++ b/api/buf.yaml @@ -1,22 +1,22 @@ version: v1 deps: - - buf.build/googleapis/googleapis:62f35d8aed1149c291d606d958a7ce32 - - buf.build/opencensus/opencensus - - buf.build/beta/prometheus - - buf.build/opentelemetry/opentelemetry - - buf.build/gogo/protobuf - - buf.build/cncf/xds +- buf.build/googleapis/googleapis:62f35d8aed1149c291d606d958a7ce32 +- buf.build/opencensus/opencensus +- buf.build/beta/prometheus +- buf.build/opentelemetry/opentelemetry +- buf.build/gogo/protobuf +- buf.build/cncf/xds breaking: ignore_unstable_packages: true use: - - FIELD_SAME_ONEOF - - FIELD_SAME_JSON_NAME - - FIELD_SAME_NAME - - FIELD_SAME_TYPE - - FIELD_SAME_LABEL - - FILE_SAME_PACKAGE - - FIELD_NO_DELETE_UNLESS_NUMBER_RESERVED - - FIELD_NO_DELETE_UNLESS_NAME_RESERVED + - FIELD_SAME_ONEOF + - FIELD_SAME_JSON_NAME + - FIELD_SAME_NAME + - FIELD_SAME_TYPE + - FIELD_SAME_LABEL + - FILE_SAME_PACKAGE + - FIELD_NO_DELETE_UNLESS_NUMBER_RESERVED + - FIELD_NO_DELETE_UNLESS_NAME_RESERVED lint: use: - IMPORT_USED diff --git a/changelogs/1.13.0.yaml b/changelogs/1.13.0.yaml index 3cf1cd1cf44f..c87cea183762 100644 --- a/changelogs/1.13.0.yaml +++ b/changelogs/1.13.0.yaml @@ -181,7 +181,7 @@ changes: added initial support for :ref:`UDP proxy `. deprecated: -- area: tracing +- area: tracing change: | The ``request_headers_for_tags`` field in :ref:`HTTP connection manager ` diff --git a/changelogs/1.19.5.yaml b/changelogs/1.19.5.yaml index 2551a80fd395..192362911c32 100644 --- a/changelogs/1.19.5.yaml +++ b/changelogs/1.19.5.yaml @@ -16,4 +16,3 @@ bug_fixes: - area: router change: | fixed CVE-2022-29227 which caused an internal redirect crash for requests with body/trailers. Envoy would previously crash in some cases when processing internal redirects for requests with bodies or trailers if the redirect prompts an Envoy-generated local reply. - diff --git a/changelogs/1.20.4.yaml b/changelogs/1.20.4.yaml index 9b64e0a99c12..07f1f0563f05 100644 --- a/changelogs/1.20.4.yaml +++ b/changelogs/1.20.4.yaml @@ -21,4 +21,3 @@ bug_fixes: - area: router change: | fixed CVE-2022-29227 which caused an internal redirect crash for requests with body/trailers. Envoy would previously crash in some cases when processing internal redirects for requests with bodies or trailers if the redirect prompts an Envoy-generated local reply. - diff --git a/changelogs/1.20.5.yaml b/changelogs/1.20.5.yaml index 28840d608589..fd4743af389b 100644 --- a/changelogs/1.20.5.yaml +++ b/changelogs/1.20.5.yaml @@ -4,4 +4,3 @@ bug_fixes: - area: docker change: | update Docker images (``distroless`` -> ``d65ac1a``) to resolve CVE issues in container packages. - diff --git a/changelogs/1.20.6.yaml b/changelogs/1.20.6.yaml index 0ecb7c0ed1b8..78e8eb86de4a 100644 --- a/changelogs/1.20.6.yaml +++ b/changelogs/1.20.6.yaml @@ -4,4 +4,3 @@ bug_fixes: - area: ci change: | fix disk space issue that have prevented publication. - diff --git a/changelogs/1.20.7.yaml b/changelogs/1.20.7.yaml index 6ac0de7a6974..34bc53be01c3 100644 --- a/changelogs/1.20.7.yaml +++ b/changelogs/1.20.7.yaml @@ -4,4 +4,3 @@ bug_fixes: - area: docker change: | update Docker images (``distroless`` -> ``49d2923f35d6``) to resolve CVE issues in container packages. - diff --git a/changelogs/1.21.2.yaml b/changelogs/1.21.2.yaml index ddc79f2e889b..3088f16e0c37 100644 --- a/changelogs/1.21.2.yaml +++ b/changelogs/1.21.2.yaml @@ -7,4 +7,3 @@ minor_behavior_changes: - area: perf change: | ssl contexts are now tracked without scan based garbage collection and greatly improved the performance on secret update. - diff --git a/changelogs/1.21.3.yaml b/changelogs/1.21.3.yaml index 2551a80fd395..192362911c32 100644 --- a/changelogs/1.21.3.yaml +++ b/changelogs/1.21.3.yaml @@ -16,4 +16,3 @@ bug_fixes: - area: router change: | fixed CVE-2022-29227 which caused an internal redirect crash for requests with body/trailers. Envoy would previously crash in some cases when processing internal redirects for requests with bodies or trailers if the redirect prompts an Envoy-generated local reply. - diff --git a/changelogs/1.21.4.yaml b/changelogs/1.21.4.yaml index 12d26a39644e..0fde0fa89c2a 100644 --- a/changelogs/1.21.4.yaml +++ b/changelogs/1.21.4.yaml @@ -9,4 +9,3 @@ bug_fixes: - area: docker change: | update Docker images (``distroless`` -> ``d65ac1a``) to resolve CVE issues in container packages. - diff --git a/changelogs/1.21.5.yaml b/changelogs/1.21.5.yaml index 8fdd33ca99b1..763d5ed61dbc 100644 --- a/changelogs/1.21.5.yaml +++ b/changelogs/1.21.5.yaml @@ -4,4 +4,3 @@ bug_fixes: - area: docker change: | update Docker images (``distroless`` -> ``49d2923f35d6``) to resolve CVE issues in container packages. - diff --git a/changelogs/1.22.1.yaml b/changelogs/1.22.1.yaml index 2551a80fd395..192362911c32 100644 --- a/changelogs/1.22.1.yaml +++ b/changelogs/1.22.1.yaml @@ -16,4 +16,3 @@ bug_fixes: - area: router change: | fixed CVE-2022-29227 which caused an internal redirect crash for requests with body/trailers. Envoy would previously crash in some cases when processing internal redirects for requests with bodies or trailers if the redirect prompts an Envoy-generated local reply. - diff --git a/changelogs/1.22.2.yaml b/changelogs/1.22.2.yaml index d96714974c4a..e4b68b85dfe3 100644 --- a/changelogs/1.22.2.yaml +++ b/changelogs/1.22.2.yaml @@ -4,4 +4,3 @@ bug_fixes: - area: ci change: | fixes/workarounds for CI that prevented publication of version 1.22.1. - diff --git a/changelogs/1.22.3.yaml b/changelogs/1.22.3.yaml index ff1c2ec6ab67..0c150fedb34b 100644 --- a/changelogs/1.22.3.yaml +++ b/changelogs/1.22.3.yaml @@ -4,4 +4,3 @@ bug_fixes: - area: docker change: | update Docker images (``distroless`` -> ``49d2923f35d6``) to resolve CVE issues in container packages. - diff --git a/changelogs/1.22.4.yaml b/changelogs/1.22.4.yaml index 148d077d6a98..181a8b9f4c5b 100644 --- a/changelogs/1.22.4.yaml +++ b/changelogs/1.22.4.yaml @@ -4,4 +4,3 @@ bug_fixes: - area: repo change: | fix version to resolve release issue. - diff --git a/changelogs/1.22.5.yaml b/changelogs/1.22.5.yaml index 078f5e6925a4..f5167cd25be3 100644 --- a/changelogs/1.22.5.yaml +++ b/changelogs/1.22.5.yaml @@ -10,4 +10,3 @@ bug_fixes: - area: transport_socket change: | fixed a bug that prevented the tcp stats to be retrieved when running on kernels different than the kernel where Envoy was built. - diff --git a/changelogs/1.23.0.yaml b/changelogs/1.23.0.yaml index 1c79156662b7..53b20fb10ff6 100644 --- a/changelogs/1.23.0.yaml +++ b/changelogs/1.23.0.yaml @@ -423,4 +423,3 @@ deprecated: change: | deprecated :ref:`inline_code `. Please use :ref:`default_source_code `. - diff --git a/changelogs/1.23.1.yaml b/changelogs/1.23.1.yaml index 2ce1355c4694..182b73722a39 100644 --- a/changelogs/1.23.1.yaml +++ b/changelogs/1.23.1.yaml @@ -4,4 +4,3 @@ bug_fixes: - area: listener change: | fixed a bug that doesn't handle of an update for a listener with IPv4-mapped address correctly and that will lead to a memory leak. - diff --git a/changelogs/1.23.2.yaml b/changelogs/1.23.2.yaml index f697487b15f3..f57623e4226f 100644 --- a/changelogs/1.23.2.yaml +++ b/changelogs/1.23.2.yaml @@ -5,4 +5,3 @@ bug_fixes: change: | fixed a bug causing response headers set by a Lua script to not be sent in the response (https://github.com/envoyproxy/envoy/issues/22401). This bug was introduced in Envoy v1.23.0. - diff --git a/changelogs/1.24.0.yaml b/changelogs/1.24.0.yaml index 86cfd82011eb..a88b0d14b0db 100644 --- a/changelogs/1.24.0.yaml +++ b/changelogs/1.24.0.yaml @@ -310,4 +310,3 @@ deprecated: :ref:`Route.typed_per_filter_config` or :ref:`WeightedCluster.ClusterWeight.typed_per_filter_config` to configure the CORS HTTP filter by the type :ref:`CorsPolicy in filter `. - diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 9e7f4441a4c3..9d0837f1630c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -26,34 +26,33 @@ bug_fixes: removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` - - - area: eds - change: | - removed ``envoy.reloadable_features.support_locality_update_on_eds_cluster_endpoints`` and legacy code paths. - - area: listener - change: | - removed ``envoy.reloadable_features.strict_check_on_ipv4_compat`` and legacy code paths. - - area: http - change: | - removed ``envoy.reloadable_features.deprecate_global_ints`` and legacy code paths. - - area: http - change: | - removed ``envoy.reloadable_features.allow_adding_content_type_in_local_replies`` and legacy code paths. - - area: http - change: | - removed ``envoy.reloadable_features.allow_upstream_inline_write`` and legacy code paths. - - area: http - change: | - removed ``envoy.reloadable_features.append_or_truncate`` and legacy code paths. - - area: http - change: | - removed ``envoy.reloadable_features.use_new_codec_wrapper`` and legacy code paths. - removed ``envoy.reloadable_features.append_to_accept_content_encoding_only_once`` and legacy code paths. - removed ``envoy.reloadable_features.http1_lazy_read_disable`` and legacy code paths. - - area: http - change: | - removed ``envoy.reloadable_features.http_100_continue_case_insensitive`` and legacy code paths. - removed ``envoy.reloadable_features.override_request_timeout_by_gateway_timeout`` and legacy code paths. +- area: eds + change: | + removed ``envoy.reloadable_features.support_locality_update_on_eds_cluster_endpoints`` and legacy code paths. +- area: listener + change: | + removed ``envoy.reloadable_features.strict_check_on_ipv4_compat`` and legacy code paths. +- area: http + change: | + removed ``envoy.reloadable_features.deprecate_global_ints`` and legacy code paths. +- area: http + change: | + removed ``envoy.reloadable_features.allow_adding_content_type_in_local_replies`` and legacy code paths. +- area: http + change: | + removed ``envoy.reloadable_features.allow_upstream_inline_write`` and legacy code paths. +- area: http + change: | + removed ``envoy.reloadable_features.append_or_truncate`` and legacy code paths. +- area: http + change: | + removed ``envoy.reloadable_features.use_new_codec_wrapper`` and legacy code paths. + removed ``envoy.reloadable_features.append_to_accept_content_encoding_only_once`` and legacy code paths. + removed ``envoy.reloadable_features.http1_lazy_read_disable`` and legacy code paths. +- area: http + change: | + removed ``envoy.reloadable_features.http_100_continue_case_insensitive`` and legacy code paths. + removed ``envoy.reloadable_features.override_request_timeout_by_gateway_timeout`` and legacy code paths. new_features: - area: generic_proxy diff --git a/configs/encapsulate_http_in_http2_connect.yaml b/configs/encapsulate_http_in_http2_connect.yaml index d4bd8f38666e..adbe66d1c3aa 100644 --- a/configs/encapsulate_http_in_http2_connect.yaml +++ b/configs/encapsulate_http_in_http2_connect.yaml @@ -1,9 +1,9 @@ # This configuration takes incoming HTTP requests on port 10000 and encapsulates it in a CONNECT # request which is sent upstream port 10001. bootstrap_extensions: - - name: envoy.bootstrap.internal_listener - typed_config: - "@type": type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener +- name: envoy.bootstrap.internal_listener + typed_config: + "@type": type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener static_resources: listeners: - name: http diff --git a/configs/internal_listener_proxy.yaml b/configs/internal_listener_proxy.yaml index 254201dd9084..945a3adc2f34 100644 --- a/configs/internal_listener_proxy.yaml +++ b/configs/internal_listener_proxy.yaml @@ -1,9 +1,9 @@ # This configuration listens on port 9999 and creates TCP connections to port 10000 using an # intermediate internal listener. bootstrap_extensions: - - name: envoy.bootstrap.internal_listener - typed_config: - "@type": type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener +- name: envoy.bootstrap.internal_listener + typed_config: + "@type": type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener static_resources: listeners: - name: ingress diff --git a/configs/terminate_http1_connect.yaml b/configs/terminate_http1_connect.yaml index 21170910ca9a..9f1c7650b358 100644 --- a/configs/terminate_http1_connect.yaml +++ b/configs/terminate_http1_connect.yaml @@ -36,9 +36,9 @@ static_resources: connect_matcher: {} headers: - - name: foo - string_match: - exact: bar + - name: foo + string_match: + exact: bar route: cluster: local_original_dst upgrade_configs: diff --git a/configs/terminate_http_in_http2_connect.yaml b/configs/terminate_http_in_http2_connect.yaml index 332b7ff4824a..ebe0867ad7cf 100644 --- a/configs/terminate_http_in_http2_connect.yaml +++ b/configs/terminate_http_in_http2_connect.yaml @@ -1,9 +1,9 @@ # This configuration terminates h2 CONNECT on port 10001 and then chains an HTTP filter that always responds with 200 using # an internal listener. bootstrap_extensions: - - name: envoy.bootstrap.internal_listener - typed_config: - "@type": type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener +- name: envoy.bootstrap.internal_listener + typed_config: + "@type": type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener static_resources: listeners: - name: listener_0 diff --git a/configs/upstream-filters.yaml b/configs/upstream-filters.yaml index d1dfdbb06a91..b6e68c8e4303 100644 --- a/configs/upstream-filters.yaml +++ b/configs/upstream-filters.yaml @@ -59,13 +59,13 @@ static_resources: http2_protocol_options: {} http_filters: - - name: buffer - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer - max_request_bytes: 5242880 - - name: envoy.filters.http.upstream_codec - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.upstream_codec.v3.UpstreamCodec + - name: buffer + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer + max_request_bytes: 5242880 + - name: envoy.filters.http.upstream_codec + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.upstream_codec.v3.UpstreamCodec transport_socket: name: envoy.transport_sockets.tls typed_config: @@ -73,7 +73,6 @@ static_resources: sni: www.envoyproxy.io layered_runtime: layers: - - name: static_layer - static_layer: - envoy.reloadable_features.allow_upstream_filters: true - + - name: static_layer + static_layer: + envoy.reloadable_features.allow_upstream_filters: true diff --git a/distribution/distros.yaml b/distribution/distros.yaml index 08fff5d5893f..6dc239ad27a1 100644 --- a/distribution/distros.yaml +++ b/distribution/distros.yaml @@ -1,4 +1,3 @@ - debian_bullseye: image: debian:bullseye-slim ext: bullseye.changes diff --git a/docs/root/configuration/http/http_filters/_include/admission-control-filter.yaml b/docs/root/configuration/http/http_filters/_include/admission-control-filter.yaml index 32d689c85f94..65d26495d8ea 100644 --- a/docs/root/configuration/http/http_filters/_include/admission-control-filter.yaml +++ b/docs/root/configuration/http/http_filters/_include/admission-control-filter.yaml @@ -1,4 +1,3 @@ - static_resources: listeners: - address: @@ -54,8 +53,8 @@ static_resources: - '*' name: local_service routes: - - match: { prefix: "/" } - route: { cluster: default_service } + - match: {prefix: "/"} + route: {cluster: default_service} clusters: - name: default_service load_assignment: diff --git a/docs/root/configuration/http/http_filters/_include/header-to-metadata-filter.yaml b/docs/root/configuration/http/http_filters/_include/header-to-metadata-filter.yaml index 9a594749b234..a273d7baea17 100644 --- a/docs/root/configuration/http/http_filters/_include/header-to-metadata-filter.yaml +++ b/docs/root/configuration/http/http_filters/_include/header-to-metadata-filter.yaml @@ -49,10 +49,10 @@ static_resources: lb_subset_config: fallback_policy: ANY_ENDPOINT subset_selectors: - - keys: - - default - - keys: - - version + - keys: + - default + - keys: + - version load_assignment: cluster_name: versioned-cluster endpoints: diff --git a/docs/root/configuration/http/http_filters/_include/local-rate-limit-global-configuration.yaml b/docs/root/configuration/http/http_filters/_include/local-rate-limit-global-configuration.yaml index 69370957efac..99e003a1a638 100644 --- a/docs/root/configuration/http/http_filters/_include/local-rate-limit-global-configuration.yaml +++ b/docs/root/configuration/http/http_filters/_include/local-rate-limit-global-configuration.yaml @@ -30,10 +30,10 @@ static_resources: numerator: 100 denominator: HUNDRED response_headers_to_add: - - append_action: OVERWRITE_IF_EXISTS_OR_ADD - header: - key: x-local-rate-limit - value: 'true' + - append_action: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-local-rate-limit + value: 'true' local_rate_limit_per_downstream_connection: false - name: envoy.filters.http.router typed_config: @@ -44,8 +44,8 @@ static_resources: - name: local_service domains: ["*"] routes: - - match: { prefix: "/" } - route: { cluster: default_service } + - match: {prefix: "/"} + route: {cluster: default_service} clusters: - name: default_service load_assignment: diff --git a/docs/root/configuration/http/http_filters/_include/local-rate-limit-route-specific-configuration.yaml b/docs/root/configuration/http/http_filters/_include/local-rate-limit-route-specific-configuration.yaml index 3f1293ebba51..163084e4be8a 100644 --- a/docs/root/configuration/http/http_filters/_include/local-rate-limit-route-specific-configuration.yaml +++ b/docs/root/configuration/http/http_filters/_include/local-rate-limit-route-specific-configuration.yaml @@ -24,8 +24,8 @@ static_resources: - name: local_service domains: ["*"] routes: - - match: { prefix: "/path/with/rate/limit" } - route: { cluster: service_protected_by_rate_limit } + - match: {prefix: "/path/with/rate/limit"} + route: {cluster: service_protected_by_rate_limit} typed_per_filter_config: envoy.filters.http.local_ratelimit: "@type": type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit @@ -45,12 +45,12 @@ static_resources: numerator: 100 denominator: HUNDRED response_headers_to_add: - - append_action: OVERWRITE_IF_EXISTS_OR_ADD - header: - key: x-local-rate-limit - value: 'true' - - match: { prefix: "/" } - route: { cluster: default_service } + - append_action: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-local-rate-limit + value: 'true' + - match: {prefix: "/"} + route: {cluster: default_service} clusters: - name: default_service load_assignment: diff --git a/docs/root/configuration/http/http_filters/_include/local-rate-limit-with-descriptors.yaml b/docs/root/configuration/http/http_filters/_include/local-rate-limit-with-descriptors.yaml index 0bb198019cbc..c8a75aeae5fb 100644 --- a/docs/root/configuration/http/http_filters/_include/local-rate-limit-with-descriptors.yaml +++ b/docs/root/configuration/http/http_filters/_include/local-rate-limit-with-descriptors.yaml @@ -24,11 +24,11 @@ static_resources: - name: local_service domains: ["*"] routes: - - match: { prefix: "/foo" } + - match: {prefix: "/foo"} route: cluster: service_protected_by_rate_limit rate_limits: - - actions: # any actions in here + - actions: # any actions in here - request_headers: header_name: x-envoy-downstream-service-cluster descriptor_key: client_cluster @@ -77,8 +77,8 @@ static_resources: max_tokens: 100 tokens_per_fill: 100 fill_interval: 60s - - match: { prefix: "/" } - route: { cluster: default_service } + - match: {prefix: "/"} + route: {cluster: default_service} clusters: - name: default_service load_assignment: diff --git a/docs/root/configuration/other_protocols/thrift_filters/_include/header-to-metadata-filter.yaml b/docs/root/configuration/other_protocols/thrift_filters/_include/header-to-metadata-filter.yaml index fbf000891bde..f8a1e6dd89e5 100644 --- a/docs/root/configuration/other_protocols/thrift_filters/_include/header-to-metadata-filter.yaml +++ b/docs/root/configuration/other_protocols/thrift_filters/_include/header-to-metadata-filter.yaml @@ -41,10 +41,10 @@ static_resources: lb_subset_config: fallback_policy: NO_FALLBACK subset_selectors: - - keys: - - default - - keys: - - version + - keys: + - default + - keys: + - version load_assignment: cluster_name: versioned-cluster endpoints: diff --git a/docs/root/intro/_include/life-of-a-request.yaml b/docs/root/intro/_include/life-of-a-request.yaml index ef67b812cd00..aa54f4453580 100644 --- a/docs/root/intro/_include/life-of-a-request.yaml +++ b/docs/root/intro/_include/life-of-a-request.yaml @@ -1,4 +1,3 @@ - static_resources: listeners: # There is a single listener bound to port 443. diff --git a/docs/root/intro/arch_overview/advanced/matching/_include/listener_complicated.yaml b/docs/root/intro/arch_overview/advanced/matching/_include/listener_complicated.yaml index 771699290b9c..7564b169114f 100644 --- a/docs/root/intro/arch_overview/advanced/matching/_include/listener_complicated.yaml +++ b/docs/root/intro/arch_overview/advanced/matching/_include/listener_complicated.yaml @@ -54,7 +54,7 @@ static_resources: "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy stat_prefix: tls cluster: some_service -# Snippet: 58-102 + # Snippet: 58-102 filter_chain_matcher: matcher_tree: input: diff --git a/docs/root/intro/arch_overview/advanced/matching/_include/listener_tls.yaml b/docs/root/intro/arch_overview/advanced/matching/_include/listener_tls.yaml index 7c7337cb04a3..2f8185f948f4 100644 --- a/docs/root/intro/arch_overview/advanced/matching/_include/listener_tls.yaml +++ b/docs/root/intro/arch_overview/advanced/matching/_include/listener_tls.yaml @@ -33,7 +33,7 @@ static_resources: "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy stat_prefix: plaintext cluster: some_service -# Snippet: 37-56 + # Snippet: 37-56 filter_chain_matcher: matcher_tree: input: diff --git a/docs/root/intro/arch_overview/advanced/matching/_include/listener_vip.yaml b/docs/root/intro/arch_overview/advanced/matching/_include/listener_vip.yaml index 261fa84469a7..d982d9f8c3b8 100644 --- a/docs/root/intro/arch_overview/advanced/matching/_include/listener_vip.yaml +++ b/docs/root/intro/arch_overview/advanced/matching/_include/listener_vip.yaml @@ -25,7 +25,7 @@ static_resources: "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy stat_prefix: default cluster: original_dst -# Snippet: 29-48 + # Snippet: 29-48 filter_chain_matcher: matcher_tree: input: diff --git a/examples/brotli/brotli-envoy.yaml b/examples/brotli/brotli-envoy.yaml index fab33e347c2e..703af847b73b 100644 --- a/examples/brotli/brotli-envoy.yaml +++ b/examples/brotli/brotli-envoy.yaml @@ -29,7 +29,7 @@ static_resources: common_config: min_content_length: 100 content_type: - - application/json + - application/json disable_on_etag_header: true compressor_library: name: text_optimized @@ -131,7 +131,7 @@ static_resources: common_config: min_content_length: 100 content_type: - - text/plain + - text/plain disable_on_etag_header: true compressor_library: name: text_optimized diff --git a/examples/kafka/envoy.yaml b/examples/kafka/envoy.yaml index f6d762e594fa..a0425882565f 100644 --- a/examples/kafka/envoy.yaml +++ b/examples/kafka/envoy.yaml @@ -2,8 +2,8 @@ static_resources: listeners: - address: socket_address: - address: 0.0.0.0 # Host that Kafka clients should connect to. - port_value: 10000 # Port that Kafka clients should connect to. + address: 0.0.0.0 # Host that Kafka clients should connect to. + port_value: 10000 # Port that Kafka clients should connect to. filter_chains: - filters: - name: envoy.filters.network.kafka_broker diff --git a/examples/local_ratelimit/ratelimit-envoy.yaml b/examples/local_ratelimit/ratelimit-envoy.yaml index 92cdde6ea62c..a81273a65ca4 100644 --- a/examples/local_ratelimit/ratelimit-envoy.yaml +++ b/examples/local_ratelimit/ratelimit-envoy.yaml @@ -41,10 +41,10 @@ static_resources: numerator: 100 denominator: HUNDRED response_headers_to_add: - - append_action: OVERWRITE_IF_EXISTS_OR_ADD - header: - key: x-local-rate-limit - value: 'true' + - append_action: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-local-rate-limit + value: 'true' local_rate_limit_per_downstream_connection: false - name: envoy.filters.http.router typed_config: @@ -90,10 +90,10 @@ static_resources: numerator: 100 denominator: HUNDRED response_headers_to_add: - - append_action: OVERWRITE_IF_EXISTS_OR_ADD - header: - key: x-local-rate-limit - value: 'true' + - append_action: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-local-rate-limit + value: 'true' local_rate_limit_per_downstream_connection: false - name: envoy.filters.http.router typed_config: diff --git a/examples/locality-load-balancing/envoy-proxy.yaml b/examples/locality-load-balancing/envoy-proxy.yaml index 66b07a09d5e2..189cee7df2cf 100644 --- a/examples/locality-load-balancing/envoy-proxy.yaml +++ b/examples/locality-load-balancing/envoy-proxy.yaml @@ -40,14 +40,14 @@ static_resources: type: STRICT_DNS lb_policy: ROUND_ROBIN health_checks: - - interval: 2s - timeout: 3s - no_traffic_interval: 4s - no_traffic_healthy_interval: 4s - unhealthy_threshold: 1 - healthy_threshold: 1 - http_health_check: - path: "/" + - interval: 2s + timeout: 3s + no_traffic_interval: 4s + no_traffic_healthy_interval: 4s + unhealthy_threshold: 1 + healthy_threshold: 1 + http_health_check: + path: "/" load_assignment: cluster_name: backend endpoints: @@ -55,55 +55,55 @@ static_resources: region: local zone: zone-1 load_balancing_weight: 1 - priority: 0 # highest + priority: 0 # highest lb_endpoints: - - endpoint: - address: - socket_address: - address: backend-local-1 - port_value: 8000 - health_check_config: + - endpoint: + address: + socket_address: + address: backend-local-1 port_value: 8000 - hostname: backend-local-1 + health_check_config: + port_value: 8000 + hostname: backend-local-1 - locality: region: local zone: zone-2 load_balancing_weight: 1 priority: 1 lb_endpoints: - - endpoint: - address: - socket_address: - address: backend-local-2 - port_value: 8000 - health_check_config: + - endpoint: + address: + socket_address: + address: backend-local-2 port_value: 8000 - hostname: backend-local-2 + health_check_config: + port_value: 8000 + hostname: backend-local-2 - locality: region: remote zone: zone-1 load_balancing_weight: 1 priority: 1 lb_endpoints: - - endpoint: - address: - socket_address: - address: backend-remote-1 - port_value: 8000 - health_check_config: + - endpoint: + address: + socket_address: + address: backend-remote-1 port_value: 8000 - hostname: backend-remote-1 + health_check_config: + port_value: 8000 + hostname: backend-remote-1 - locality: region: remote zone: zone-2 load_balancing_weight: 1 priority: 2 lb_endpoints: - - endpoint: - address: - socket_address: - address: backend-remote-2 - port_value: 8000 - health_check_config: + - endpoint: + address: + socket_address: + address: backend-remote-2 port_value: 8000 - hostname: backend-remote-2 + health_check_config: + port_value: 8000 + hostname: backend-remote-2 diff --git a/examples/wasm-cc/docker-compose.yaml b/examples/wasm-cc/docker-compose.yaml index 8080ddeef42f..b514110f2cfd 100644 --- a/examples/wasm-cc/docker-compose.yaml +++ b/examples/wasm-cc/docker-compose.yaml @@ -12,6 +12,6 @@ services: web_service: environment: - - PORT=9000 + - PORT=9000 build: context: ../shared/echo diff --git a/examples/zstd/zstd-envoy.yaml b/examples/zstd/zstd-envoy.yaml index 4656a6349eb5..b3862468db75 100644 --- a/examples/zstd/zstd-envoy.yaml +++ b/examples/zstd/zstd-envoy.yaml @@ -29,7 +29,7 @@ static_resources: common_config: min_content_length: 100 content_type: - - application/json + - application/json disable_on_etag_header: true compressor_library: name: text_optimized @@ -131,7 +131,7 @@ static_resources: common_config: min_content_length: 100 content_type: - - text/plain + - text/plain disable_on_etag_header: true compressor_library: name: text_optimized diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index ed3aecce4bae..e0fcb8dd201d 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -771,21 +771,21 @@ envoy.matching.matchers.ip: type_urls: - envoy.extensions.matching.input_matchers.ip.v3.Ip envoy.path.match.uri_template.uri_template_matcher: - categories: - - envoy.path.match - security_posture: robust_to_untrusted_downstream_and_upstream - status: stable - undocumented: true - type_urls: - - envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig + categories: + - envoy.path.match + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable + undocumented: true + type_urls: + - envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig envoy.path.rewrite.uri_template.uri_template_rewriter: - categories: - - envoy.path.rewrite - security_posture: robust_to_untrusted_downstream_and_upstream - status: stable - undocumented: true - type_urls: - - envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig + categories: + - envoy.path.rewrite + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable + undocumented: true + type_urls: + - envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig envoy.quic.proof_source.filter_chain: categories: - envoy.quic.proof_source @@ -1096,17 +1096,17 @@ envoy.wasm.runtime.v8: envoy.wasm.runtime.wamr: categories: - envoy.wasm.runtime - security_posture: unknown # "This may never change from unknown until the threat model at https://envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/threat_model#core-and-extensions is updated to capture additional Wasm runtimes". + security_posture: unknown # "This may never change from unknown until the threat model at https://envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/threat_model#core-and-extensions is updated to capture additional Wasm runtimes". status: alpha envoy.wasm.runtime.wasmtime: categories: - envoy.wasm.runtime - security_posture: unknown # "This may never change from unknown until the threat model at https://envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/threat_model#core-and-extensions is updated to capture additional Wasm runtimes". + security_posture: unknown # "This may never change from unknown until the threat model at https://envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/threat_model#core-and-extensions is updated to capture additional Wasm runtimes". status: alpha envoy.wasm.runtime.wavm: categories: - envoy.wasm.runtime - security_posture: unknown # "This may never change from unknown until the threat model at https://envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/threat_model#core-and-extensions is updated to capture additional Wasm runtimes". + security_posture: unknown # "This may never change from unknown until the threat model at https://envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/threat_model#core-and-extensions is updated to capture additional Wasm runtimes". status: alpha envoy.watchdog.profile_action: categories: diff --git a/test/config/integration/server.yaml b/test/config/integration/server.yaml index e35a6acbf0c8..ff742cdbebff 100644 --- a/test/config/integration/server.yaml +++ b/test/config/integration/server.yaml @@ -4,7 +4,7 @@ static_resources: socket_address: address: "{{ ip_loopback_address }}" port_value: 0 - enable_reuse_port: {{ enable_reuse_port }} + enable_reuse_port: "{{ enable_reuse_port }}" filter_chains: - filters: - name: http diff --git a/test/config/integration/server_multiple_addresses.yaml b/test/config/integration/server_multiple_addresses.yaml index e38216f5a435..20d61e64dce0 100644 --- a/test/config/integration/server_multiple_addresses.yaml +++ b/test/config/integration/server_multiple_addresses.yaml @@ -9,7 +9,7 @@ static_resources: socket_address: address: "{{ ip_loopback_address }}" port_value: 0 - enable_reuse_port: {{ enable_reuse_port }} + enable_reuse_port: "{{ enable_reuse_port }}" filter_chains: - filters: - name: http diff --git a/test/config/integration/server_xds.lds.yaml b/test/config/integration/server_xds.lds.yaml index 9e92a1ee6fb0..78c3238302f5 100644 --- a/test/config/integration/server_xds.lds.yaml +++ b/test/config/integration/server_xds.lds.yaml @@ -24,4 +24,3 @@ resources: - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - diff --git a/test/extensions/filters/http/ext_authz/ext_authz.yaml b/test/extensions/filters/http/ext_authz/ext_authz.yaml index 19916536b4a3..c777d971d1aa 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz.yaml +++ b/test/extensions/filters/http/ext_authz/ext_authz.yaml @@ -1,76 +1,76 @@ # Regression test for https://github.com/envoyproxy/envoy/issues/17344 static_resources: listeners: - - address: - socket_address: - address: 0.0.0.0 - port_value: 8080 - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress_http - route_config: - name: local_route - virtual_hosts: - - name: local_service - domains: ["*"] - routes: - - match: - prefix: "/" - route: - cluster: local_service - http_filters: - - name: envoy.ext_authz - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz - failure_mode_allow: false - transport_api_version: V3 - status_on_error: - code: 503 - grpc_service: - envoy_grpc: - cluster_name: ext_authz-service - timeout: 0.5s - with_request_body: - max_request_bytes: 10240 - allow_partial_message: true - pack_as_bytes: false - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + - address: + socket_address: + address: 0.0.0.0 + port_value: 8080 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: local_service + http_filters: + - name: envoy.ext_authz + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz + failure_mode_allow: false + transport_api_version: V3 + status_on_error: + code: 503 + grpc_service: + envoy_grpc: + cluster_name: ext_authz-service + timeout: 0.5s + with_request_body: + max_request_bytes: 10240 + allow_partial_message: true + pack_as_bytes: false + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router clusters: - - name: local_service - connect_timeout: 30s - type: STRICT_DNS - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: local_service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: main - port_value: 8080 - - name: ext_authz-service - type: STRICT_DNS - lb_policy: ROUND_ROBIN - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {} - load_assignment: - cluster_name: ext_authz-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: opa - port_value: 80 + - name: local_service + connect_timeout: 30s + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: local_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: main + port_value: 8080 + - name: ext_authz-service + type: STRICT_DNS + lb_policy: ROUND_ROBIN + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: ext_authz-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: opa + port_value: 80 admin: address: socket_address: diff --git a/test/server/test_data/server/access_log_filter_bootstrap.yaml b/test/server/test_data/server/access_log_filter_bootstrap.yaml index ffe1133512a7..92c05b6c6946 100644 --- a/test/server/test_data/server/access_log_filter_bootstrap.yaml +++ b/test/server/test_data/server/access_log_filter_bootstrap.yaml @@ -5,7 +5,7 @@ admin: "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog path: "{{ null_device_path }}" filter: - not_health_check_filter: {} + not_health_check_filter: {} address: socket_address: address: "{{ ntop_ip_loopback_address }}" diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index e344e65a757b..04e1647c4869 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -217,13 +217,13 @@ paths: - test/test_common/wasm_base.h dir_order: - - envoy - - common - - source - - exe - - server - - extensions - - test +- envoy +- common +- source +- exe +- server +- extensions +- test re: codeowners_contrib: (/contrib/[^@]*\s+)(@.*) diff --git a/tools/dependency/cve.yaml b/tools/dependency/cve.yaml index dfe60fad67bb..8fff58bc9ed7 100644 --- a/tools/dependency/cve.yaml +++ b/tools/dependency/cve.yaml @@ -1,4 +1,3 @@ - # We only look back a few years, since we shouldn't have any ancient deps. start_year: 2018 diff --git a/tools/extensions/extensions_schema.yaml b/tools/extensions/extensions_schema.yaml index d51644e2a696..d85c9ac503d6 100644 --- a/tools/extensions/extensions_schema.yaml +++ b/tools/extensions/extensions_schema.yaml @@ -1,4 +1,3 @@ - builtin: - envoy.request_id.uuid - envoy.upstreams.tcp.generic From c827ba7546073034dd6990247f5c8d5b6ec3f102 Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Tue, 1 Nov 2022 09:16:48 -0400 Subject: [PATCH 062/112] tests: ensuring unified mux is correctly enabled/disabled (#23750) Ensuring that when setting the envoy.reloadable_features.unified_mux runtime flag to either true or false will be reflected correctly in the tests. Risk Level: Low - tests only. Testing: Updating tests. Signed-off-by: Adi Suissa-Peleg --- .../config/subscription_factory_impl_test.cc | 8 +++--- ...mum_clusters_validator_integration_test.cc | 9 ++++--- test/integration/ads_integration.cc | 9 ++++--- test/integration/ads_integration_test.cc | 27 ++++++++++--------- test/integration/cds_integration_test.cc | 9 ++++--- test/integration/rtds_integration_test.cc | 9 ++++--- test/integration/scoped_rds.h | 9 ++++--- test/integration/vhds.h | 5 ++-- test/integration/vhds_integration_test.cc | 5 ++-- ...xds_delegate_extension_integration_test.cc | 5 ++-- test/server/config_validation/xds_fuzz.cc | 5 ++-- 11 files changed, 52 insertions(+), 48 deletions(-) diff --git a/test/common/config/subscription_factory_impl_test.cc b/test/common/config/subscription_factory_impl_test.cc index 89a49e42e069..e1e712da8aea 100644 --- a/test/common/config/subscription_factory_impl_test.cc +++ b/test/common/config/subscription_factory_impl_test.cc @@ -84,9 +84,8 @@ class SubscriptionFactoryTestUnifiedOrLegacyMux public testing::WithParamInterface { public: SubscriptionFactoryTestUnifiedOrLegacyMux() { - if (GetParam() == LegacyOrUnified::Unified) { - scoped_runtime_.mergeValues({{"envoy.reloadable_features.unified_mux", "true"}}); - } + scoped_runtime_.mergeValues({{"envoy.reloadable_features.unified_mux", + (GetParam() == LegacyOrUnified::Unified) ? "true" : "false"}}); } TestScopedRuntime scoped_runtime_; @@ -378,7 +377,8 @@ TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, "xdstp://foo/envoy.config.endpoint.v3.ClusterLoadAssignment/bar", config) ->start({}), EnvoyException, - "Missing or not supported config source specifier in envoy::config::core::v3::ConfigSource " + "Missing or not supported config source specifier in " + "envoy::config::core::v3::ConfigSource " "for a collection. Only ADS and gRPC in delta-xDS mode are supported."); } diff --git a/test/extensions/config/validators/minimum_clusters/minimum_clusters_validator_integration_test.cc b/test/extensions/config/validators/minimum_clusters/minimum_clusters_validator_integration_test.cc index d79040896190..087b5833d39f 100644 --- a/test/extensions/config/validators/minimum_clusters/minimum_clusters_validator_integration_test.cc +++ b/test/extensions/config/validators/minimum_clusters/minimum_clusters_validator_integration_test.cc @@ -36,10 +36,11 @@ class MinimumClustersValidatorIntegrationTest : public Grpc::DeltaSotwIntegratio sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw ? "GRPC" : "DELTA_GRPC")) { - if (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || - sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || + sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) + ? "true" + : "false"); use_lds_ = false; sotw_or_delta_ = sotwOrDelta(); } diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index a31ecdd4551c..8447e321f65f 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -33,10 +33,11 @@ AdsIntegrationTest::AdsIntegrationTest() // 'ads_cluster'. skip_tag_extraction_rule_check_ = true; - if (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || - sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || + sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) + ? "true" + : "false"); use_lds_ = false; create_xds_upstream_ = true; tls_xds_upstream_ = true; diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index e2f57498bdd9..26edfafd3ad1 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -1245,10 +1245,11 @@ class AdsFailIntegrationTest : public AdsDeltaSotwIntegrationSubStateParamTest, // stat_prefix 'ads_cluster'. skip_tag_extraction_rule_check_ = true; - if (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || - sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || + sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) + ? "true" + : "false"); create_xds_upstream_ = true; use_lds_ = false; sotw_or_delta_ = sotwOrDelta(); @@ -1299,10 +1300,11 @@ class AdsConfigIntegrationTest : public AdsDeltaSotwIntegrationSubStateParamTest // 'ads_cluster'. skip_tag_extraction_rule_check_ = true; - if (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || - sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || + sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) + ? "true" + : "false"); create_xds_upstream_ = true; use_lds_ = false; sotw_or_delta_ = sotwOrDelta(); @@ -1499,10 +1501,11 @@ class AdsClusterFromFileIntegrationTest : public AdsDeltaSotwIntegrationSubState // 'ads_cluster'. skip_tag_extraction_rule_check_ = true; - if (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || - sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || + sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) + ? "true" + : "false"); create_xds_upstream_ = true; use_lds_ = false; sotw_or_delta_ = sotwOrDelta(); diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index 9c741350d42a..8e03b0421201 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -39,10 +39,11 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht ? "GRPC" : "DELTA_GRPC")), cluster_creator_(&ConfigHelper::buildStaticCluster) { - if (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || - sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || + sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) + ? "true" + : "false"); use_lds_ = false; sotw_or_delta_ = sotwOrDelta(); } diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc index 7353e1f75ac7..1b3c55c7b71d 100644 --- a/test/integration/rtds_integration_test.cc +++ b/test/integration/rtds_integration_test.cc @@ -89,10 +89,11 @@ class RtdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public H sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw ? "GRPC" : "DELTA_GRPC")) { - if (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || - sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || + sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) + ? "true" + : "false"); use_lds_ = false; create_xds_upstream_ = true; sotw_or_delta_ = sotwOrDelta(); diff --git a/test/integration/scoped_rds.h b/test/integration/scoped_rds.h index b933075025e2..fb354c726460 100644 --- a/test/integration/scoped_rds.h +++ b/test/integration/scoped_rds.h @@ -35,10 +35,11 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, // 'srds_cluster'. skip_tag_extraction_rule_check_ = true; - if (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || - sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + (sotwOrDelta() == Grpc::SotwOrDelta::UnifiedSotw || + sotwOrDelta() == Grpc::SotwOrDelta::UnifiedDelta) + ? "true" + : "false"); } ~ScopedRdsIntegrationTest() override { resetConnections(); } diff --git a/test/integration/vhds.h b/test/integration/vhds.h index e3ce9aea1666..d62d3947d857 100644 --- a/test/integration/vhds.h +++ b/test/integration/vhds.h @@ -143,9 +143,8 @@ class VhdsIntegrationTest : public HttpIntegrationTest, public: VhdsIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion(), config()) { use_lds_ = false; - if (isUnified()) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + isUnified() ? "true" : "false"); } void TearDown() override { cleanUpXdsConnection(); } diff --git a/test/integration/vhds_integration_test.cc b/test/integration/vhds_integration_test.cc index d9b77633bdb1..de5fec0635bf 100644 --- a/test/integration/vhds_integration_test.cc +++ b/test/integration/vhds_integration_test.cc @@ -41,9 +41,8 @@ class VhdsInitializationTest : public HttpIntegrationTest, public: VhdsInitializationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion(), config()) { use_lds_ = false; - if (isUnified()) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + isUnified() ? "true" : "false"); } void TearDown() override { cleanUpXdsConnection(); } diff --git a/test/integration/xds_delegate_extension_integration_test.cc b/test/integration/xds_delegate_extension_integration_test.cc index 577fc7a36367..28d0ddc8cbbc 100644 --- a/test/integration/xds_delegate_extension_integration_test.cc +++ b/test/integration/xds_delegate_extension_integration_test.cc @@ -102,9 +102,8 @@ class XdsDelegateExtensionIntegrationTest : public Grpc::UnifiedOrLegacyMuxInteg use_lds_ = false; create_xds_upstream_ = true; - if (isUnified()) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + isUnified() ? "true" : "false"); // Make the default cluster HTTP2. config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { diff --git a/test/server/config_validation/xds_fuzz.cc b/test/server/config_validation/xds_fuzz.cc index 28f239d89325..9b91aaae53b7 100644 --- a/test/server/config_validation/xds_fuzz.cc +++ b/test/server/config_validation/xds_fuzz.cc @@ -64,9 +64,8 @@ XdsFuzzTest::XdsFuzzTest(const test::server::config_validation::XdsTestCase& inp : "DELTA_GRPC")), verifier_(input.config().sotw_or_delta()), actions_(input.actions()), version_(1), ip_version_(TestEnvironment::getIpVersionsForTest()[0]) { - if (use_unified_mux) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); - } + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", + use_unified_mux ? "true" : "false"); use_lds_ = false; create_xds_upstream_ = true; tls_xds_upstream_ = false; From 44799aaffd3aa5b95423ddebd8d095aa47eb0f5c Mon Sep 17 00:00:00 2001 From: Bo-Cheng Chu Date: Tue, 1 Nov 2022 06:44:16 -0700 Subject: [PATCH 063/112] Update jwt_verify_lib to latest commit (#23763) update repo location Signed-off-by: bochengchu --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 0da3f798ac58..846e9929c732 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -748,13 +748,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "jwt_verify_lib", project_desc = "JWT verification library for C++", project_url = "https://github.com/google/jwt_verify_lib", - version = "26c22c0ce1bc607eec8fa5dd26b707378adc7a88", - sha256 = "8964c2b3a833dc5fc2600b2768ea1e73a0fcf8a1ed9d2cbc5fa3387c4cdd5caa", + version = "fd3d4d61835f7c35274cb8e6aba923542ae52f0b", + sha256 = "87a6ad666b4160becd1f839483e00a6ea3c46cb0fa9d8e6566a5fc9f315201be", strip_prefix = "jwt_verify_lib-{version}", urls = ["https://github.com/google/jwt_verify_lib/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.http.jwt_authn", "envoy.filters.http.gcp_authn"], - release_date = "2022-09-22", + release_date = "2022-10-28", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/google/jwt_verify_lib/blob/{version}/LICENSE", From bb4fd0141017ee85d3d78fefb606b37c96a8afd6 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 1 Nov 2022 07:16:25 -0700 Subject: [PATCH 064/112] deps: update googleurl (#23722) This slightly improves our patch because upstream started validating __has_attribute exists itself now. There is a little bit of new patching due to https://quiche-review.googlesource.com/c/googleurl/+/11180 Fixes #23340 Signed-off-by: Keith Smiley keithbsmiley@gmail.com --- bazel/external/googleurl.patch | 198 +++++++++++++++++---------------- bazel/repository_locations.bzl | 8 +- 2 files changed, 105 insertions(+), 101 deletions(-) diff --git a/bazel/external/googleurl.patch b/bazel/external/googleurl.patch index 57c51272c32d..81d35d7d612b 100644 --- a/bazel/external/googleurl.patch +++ b/bazel/external/googleurl.patch @@ -2,13 +2,13 @@ # project using clang-cl. Tracked in https://github.com/envoyproxy/envoy/issues/11974. diff --git a/base/compiler_specific.h b/base/compiler_specific.h -index 6651220..a469c19 100644 +index 0174b6d..fb5b80d 100644 --- a/base/compiler_specific.h +++ b/base/compiler_specific.h @@ -7,10 +7,6 @@ - + #include "build/build_config.h" - + -#if defined(COMPILER_MSVC) && !defined(__clang__) -#error "Only clang-cl is supported on Windows, see https://crbug.com/988071" -#endif @@ -16,114 +16,118 @@ index 6651220..a469c19 100644 // This is a wrapper around `__has_cpp_attribute`, which can be used to test for // the presence of an attribute. In case the compiler does not support this // macro it will simply evaluate to 0. -@@ -75,8 +71,12 @@ - // prevent code folding, see NO_CODE_FOLDING() in base/debug/alias.h. - // Use like: - // void NOT_TAIL_CALLED FooBar(); --#if defined(__clang__) && __has_attribute(not_tail_called) -+#if defined(__clang__) -+#if defined(__has_attribute) -+#if __has_attribute(not_tail_called) - #define NOT_TAIL_CALLED __attribute__((not_tail_called)) -+#endif -+#endif - #else - #define NOT_TAIL_CALLED - #endif -@@ -273,7 +273,9 @@ - #endif - #endif - --#if defined(__clang__) && __has_attribute(uninitialized) -+#if defined(__clang__) -+#if defined(__has_attribute) -+#if __has_attribute(uninitialized) - // Attribute "uninitialized" disables -ftrivial-auto-var-init=pattern for - // the specified variable. - // Library-wide alternative is -@@ -304,6 +306,8 @@ - // E.g. platform, bot, benchmark or test name in patch description or next to - // the attribute. - #define STACK_UNINITIALIZED __attribute__((uninitialized)) -+#endif -+#endif - #else - #define STACK_UNINITIALIZED - #endif -@@ -365,8 +369,12 @@ inline constexpr bool AnalyzerAssumeTrue(bool arg) { - #endif // defined(__clang_analyzer__) - - // Use nomerge attribute to disable optimization of merging multiple same calls. --#if defined(__clang__) && __has_attribute(nomerge) -+#if defined(__clang__) -+#if defined(__has_attribute) -+#if __has_attribute(nomerge) - #define NOMERGE [[clang::nomerge]] -+#endif -+#endif - #else - #define NOMERGE - #endif -@@ -392,8 +400,12 @@ inline constexpr bool AnalyzerAssumeTrue(bool arg) { - // See also: - // https://clang.llvm.org/docs/AttributeReference.html#trivial-abi - // https://libcxx.llvm.org/docs/DesignDocs/UniquePtrTrivialAbi.html --#if defined(__clang__) && __has_attribute(trivial_abi) -+#if defined(__clang__) -+#if defined(__has_attribute) -+#if __has_attribute(trivial_abi) - #define TRIVIAL_ABI [[clang::trivial_abi]] -+#endif -+#endif - #else - #define TRIVIAL_ABI - #endif -@@ -401,8 +413,12 @@ inline constexpr bool AnalyzerAssumeTrue(bool arg) { - // Marks a member function as reinitializing a moved-from variable. - // See also - // https://clang.llvm.org/extra/clang-tidy/checks/bugprone-use-after-move.html#reinitialization --#if defined(__clang__) && __has_attribute(reinitializes) -+#if defined(__clang__) -+#if defined(__has_attribute) -+#if __has_attribute(reinitializes) - #define REINITIALIZES_AFTER_MOVE [[clang::reinitializes]] -+#endif -+#endif - #else - #define REINITIALIZES_AFTER_MOVE +@@ -398,7 +394,7 @@ inline constexpr bool AnalyzerAssumeTrue(bool arg) { + #define CONSTINIT #endif -# TODO(keith): Remove once bazel supports newer NDK versions https://github.com/bazelbuild/bazel/issues/12889 - +-#if defined(__clang__) ++#if defined(__clang__) && HAS_CPP_ATTRIBUTE(gsl::Pointer) + #define GSL_OWNER [[gsl::Owner]] + #define GSL_POINTER [[gsl::Pointer]] + #else diff --git a/base/containers/checked_iterators.h b/base/containers/checked_iterators.h -index b5fe925..31aa81e 100644 +index dc8d2ba..9306697 100644 --- a/base/containers/checked_iterators.h +++ b/base/containers/checked_iterators.h @@ -237,9 +237,11 @@ using CheckedContiguousConstIterator = CheckedContiguousIterator; // [3] https://wg21.link/pointer.traits.optmem namespace std { - + +#ifdef SUPPORTS_CPP_17_CONTIGUOUS_ITERATOR template struct __is_cpp17_contiguous_iterator<::gurl_base::CheckedContiguousIterator> : true_type {}; +#endif - + template struct pointer_traits<::gurl_base::CheckedContiguousIterator> { -# TODO(keith): Remove once https://quiche-review.googlesource.com/c/googleurl/+/10980 lands +# TODO(keith): Remove unused parameter workarounds when https://quiche-review.googlesource.com/c/googleurl/+/11180 lands -diff --git a/base/compiler_specific.h b/base/compiler_specific.h -index 3a85453..a329de2 100644 ---- a/base/compiler_specific.h -+++ b/base/compiler_specific.h -@@ -382,7 +382,7 @@ inline constexpr bool AnalyzerAssumeTrue(bool arg) { - #define CONSTINIT - #endif - --#if defined(__clang__) -+#if defined(__clang__) && HAS_CPP_ATTRIBUTE(gsl::Pointer) - #define GSL_OWNER [[gsl::Owner]] - #define GSL_POINTER [[gsl::Pointer]] - #else +diff --git a/base/numerics/clamped_math_impl.h b/base/numerics/clamped_math_impl.h +index 10023f0..783f5da 100644 +--- a/base/numerics/clamped_math_impl.h ++++ b/base/numerics/clamped_math_impl.h +@@ -36,6 +36,7 @@ template ::value && + !std::is_signed::value>::type* = nullptr> + constexpr T SaturatedNegWrapper(T value) { ++ (void)value; // unused + return T(0); + } + +diff --git a/base/numerics/safe_conversions.h b/base/numerics/safe_conversions.h +index 4a9494e..ba44fa0 100644 +--- a/base/numerics/safe_conversions.h ++++ b/base/numerics/safe_conversions.h +@@ -45,6 +45,7 @@ template + struct IsValueInRangeFastOp { + static constexpr bool is_supported = false; + static constexpr bool Do(Src value) { ++ (void)value; // unused + // Force a compile failure if instantiated. + return CheckOnFailure::template HandleFailure(); + } +@@ -164,6 +165,7 @@ template + struct SaturateFastOp { + static constexpr bool is_supported = false; + static constexpr Dst Do(Src value) { ++ (void)value; // unused + // Force a compile failure if instantiated. + return CheckOnFailure::template HandleFailure(); + } +diff --git a/build_config/build_config.bzl b/build_config/build_config.bzl +index 5960d2a..08295ff 100644 +--- a/build_config/build_config.bzl ++++ b/build_config/build_config.bzl +@@ -7,6 +7,7 @@ _default_copts = select({ + "//conditions:default": [ + "-std=c++17", + "-fno-strict-aliasing", ++ "-Wno-unused-parameter", + ], + }) + +diff --git a/url/url_canon_internal.h b/url/url_canon_internal.h +index 58ae144..467da0b 100644 +--- a/url/url_canon_internal.h ++++ b/url/url_canon_internal.h +@@ -305,6 +305,7 @@ inline bool AppendUTF8EscapedChar(const char* str, + // through it will point to the next character to be considered. On failure, + // |*begin| will be unchanged. + inline bool Is8BitChar(char c) { ++ (void)c; // unused + return true; // this case is specialized to avoid a warning + } + inline bool Is8BitChar(char16_t c) { + +# TODO(keith): Remove when https://quiche-review.googlesource.com/c/googleurl/+/11200 lands + +diff --git a/base/memory/raw_ptr_exclusion.h b/base/memory/raw_ptr_exclusion.h +index f881c04..4e4f7df 100644 +--- a/base/memory/raw_ptr_exclusion.h ++++ b/base/memory/raw_ptr_exclusion.h +@@ -8,7 +8,7 @@ + #include "polyfills/base/allocator/buildflags.h" + #include "build/build_config.h" + +-#if defined(OFFICIAL_BUILD) && !BUILDFLAG(FORCE_ENABLE_RAW_PTR_EXCLUSION) ++#if !defined(__clang__) || (defined(OFFICIAL_BUILD) && !BUILDFLAG(FORCE_ENABLE_RAW_PTR_EXCLUSION)) + // The annotation changed compiler output and increased binary size so disable + // for official builds. + // TODO(crbug.com/1320670): Remove when issue is resolved. + +# TODO(keith): Remove when https://quiche-review.googlesource.com/c/googleurl/+/11220 lands + +diff --git a/base/BUILD b/base/BUILD +index 60ca578..6a793a6 100644 +--- a/base/BUILD ++++ b/base/BUILD +@@ -55,6 +55,7 @@ cc_library( + "strings/string_number_conversions.h", + "strings/utf_string_conversions.h", + "strings/utf_string_conversion_utils.h", ++ "win/win_handle_types.h", + ] + build_config.strings_hdrs, + copts = build_config.default_copts, + textual_hdrs = [ diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 846e9929c732..1e9427c12750 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1094,10 +1094,10 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Chrome URL parsing library", project_desc = "Chrome URL parsing library", project_url = "https://quiche.googlesource.com/googleurl", - # Static snapshot of https://quiche.googlesource.com/googleurl/+archive/9cdb1f4d1a365ebdbcbf179dadf7f8aa5ee802e7.tar.gz. - version = "9cdb1f4d1a365ebdbcbf179dadf7f8aa5ee802e7", - sha256 = "a1bc96169d34dcc1406ffb750deef3bc8718bd1f9069a2878838e1bd905de989", - urls = ["https://storage.googleapis.com/quiche-envoy-integration/googleurl_{version}.tar.gz"], + # Static snapshot of https://quiche.googlesource.com/googleurl/+archive/fd287250b7f0d876478d88dd7641ba8a2e130bd2.tar.gz + version = "fd287250b7f0d876478d88dd7641ba8a2e130bd2", + sha256 = "053e6d8c80c7c4159012254de72ec17cc67a9945e709fbe9ac4925afcdb40884", + urls = ["https://storage.googleapis.com/quiche-envoy-integration/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], extensions = [], release_date = "2022-04-04", From 321fcb2770513a430ac80cbb42bb429b958350c8 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 1 Nov 2022 11:33:59 -0400 Subject: [PATCH 065/112] build: making alwayslink optional in Enovy CC libraries (#23676) This adds a variable to change Envoy library defaults to not always link, while fixing up Envoy libraries using singletons and factory registrations to still always link. Risk Level: high Testing: using existing tests Docs Changes: n/a Release Notes: yes Platform Specific Features: no Part of envoyproxy/envoy-mobile#175 Signed-off-by: Alyssa Wilk --- bazel/BUILD | 5 +++++ bazel/README.md | 1 + bazel/envoy_library.bzl | 14 +++++++++++++- changelogs/current.yaml | 3 +++ ci/do_ci.sh | 5 ++++- ci/osx-build-config/extensions_build_config.bzl | 4 ++++ contrib/generic_proxy/filters/network/source/BUILD | 1 + source/common/common/BUILD | 5 +++++ source/common/crypto/BUILD | 2 ++ source/common/grpc/BUILD | 5 +++++ source/common/http/match_delegate/BUILD | 5 +++++ source/common/http/matching/BUILD | 5 +++++ source/common/network/BUILD | 7 +++++++ source/common/network/matching/BUILD | 5 +++++ source/common/quic/BUILD | 5 +++++ source/common/router/BUILD | 6 ++++++ source/common/ssl/matching/BUILD | 5 +++++ source/common/upstream/BUILD | 9 +++++++++ source/common/watchdog/BUILD | 5 +++++ source/extensions/access_loggers/wasm/BUILD | 1 + .../extensions/bootstrap/internal_listener/BUILD | 5 +++-- source/extensions/common/wasm/BUILD | 6 +++++- source/extensions/common/wasm/ext/BUILD | 7 +++++++ source/extensions/extensions_build_config.bzl | 4 ++++ source/extensions/filters/http/wasm/BUILD | 1 + .../extensions/filters/network/dubbo_proxy/BUILD | 2 ++ .../filters/network/dubbo_proxy/router/BUILD | 1 + .../extensions/filters/network/thrift_proxy/BUILD | 8 ++++++++ source/extensions/filters/network/wasm/BUILD | 1 + source/extensions/grpc_credentials/example/BUILD | 1 + .../extensions/quic/connection_id_generator/BUILD | 5 +++++ source/extensions/quic/crypto_stream/BUILD | 6 ++++++ source/extensions/quic/proof_source/BUILD | 5 +++++ source/extensions/stat_sinks/wasm/BUILD | 2 ++ .../transport_sockets/tls/cert_validator/BUILD | 1 + .../extensions/access_loggers/wasm/test_data/BUILD | 9 +++++++-- test/extensions/bootstrap/wasm/test_data/BUILD | 8 ++++---- test/extensions/common/wasm/test_data/BUILD | 8 ++++---- test/extensions/filters/http/wasm/test_data/BUILD | 4 ++-- .../filters/network/wasm/test_data/BUILD | 4 ++-- test/extensions/stats_sinks/wasm/BUILD | 1 + test/extensions/stats_sinks/wasm/test_data/BUILD | 4 ++-- 42 files changed, 170 insertions(+), 21 deletions(-) diff --git a/bazel/BUILD b/bazel/BUILD index c8bb5f64da90..146f74bdd522 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -291,6 +291,11 @@ config_setting( values = {"define": "signal_trace=disabled"}, ) +config_setting( + name = "disable_library_autolink", + values = {"define": "library_autolink=disabled"}, +) + config_setting( name = "disable_object_dump_on_signal_trace", values = {"define": "object_dump_on_signal_trace=disabled"}, diff --git a/bazel/README.md b/bazel/README.md index 4e8322155508..2d238c7cc34d 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -687,6 +687,7 @@ The following optional features can be disabled on the Bazel build command-line: tcmalloc with `--define tcmalloc=gperftools` which is the default for builds other than x86_64 and aarch64. * deprecated features with `--define deprecated_features=disabled` * http3/quic with --//bazel:http3=False +* autolinking libraries with --define=library_autolink=disabled * admin HTML home page with `--define=admin_html=disabled` ## Enabling optional features diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index 12487aee0bd2..cce13bd8e97a 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -57,6 +57,7 @@ def envoy_cc_extension( tags = [], extra_visibility = [], visibility = EXTENSION_CONFIG_VISIBILITY, + alwayslink = 1, **kwargs): if "//visibility:public" not in visibility: visibility = visibility + extra_visibility @@ -66,6 +67,7 @@ def envoy_cc_extension( name = name, tags = tags, visibility = visibility, + alwayslink = alwayslink, **kwargs ) native.cc_library( @@ -83,6 +85,7 @@ def envoy_cc_contrib_extension( tags = [], extra_visibility = [], visibility = CONTRIB_EXTENSION_PACKAGE_VISIBILITY, + alwayslink = 1, **kwargs): envoy_cc_extension(name, tags, extra_visibility, visibility, **kwargs) @@ -101,10 +104,19 @@ def envoy_cc_library( strip_include_prefix = None, include_prefix = None, textual_hdrs = None, + alwayslink = None, defines = []): if tcmalloc_dep: deps += tcmalloc_external_deps(repository) + # If alwayslink is not specified, allow turning it off via --define=library_autolink=disabled + # alwayslink is defaulted on for envoy_cc_extensions to ensure the REGISTRY macros work. + if alwayslink == None: + alwayslink = select({ + repository + "//bazel:disable_library_autolink": 0, + "//conditions:default": 1, + }) + native.cc_library( name = name, srcs = srcs, @@ -122,7 +134,7 @@ def envoy_cc_library( envoy_external_dep_path("abseil_strings"), envoy_external_dep_path("fmtlib"), ], - alwayslink = 1, + alwayslink = alwayslink, linkstatic = envoy_linkstatic(), strip_include_prefix = strip_include_prefix, include_prefix = include_prefix, diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 9d0837f1630c..fec25a1a3dfd 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -55,6 +55,9 @@ removed_config_or_runtime: removed ``envoy.reloadable_features.override_request_timeout_by_gateway_timeout`` and legacy code paths. new_features: +- area: build + change: | + added an option ``--define=library_autolink=disabled`` to disable autolinking libraries. - area: generic_proxy change: | added :ref:`dubbo codec support ` to the diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 70a2f4726d79..b179df0c8e0a 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -286,7 +286,10 @@ elif [[ "$CI_TARGET" == "bazel.gcc" ]]; then elif [[ "$CI_TARGET" == "bazel.debug" ]]; then setup_clang_toolchain echo "Testing ${TEST_TARGETS[*]}" - bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" + # Make sure that there are no regressions to building Envoy with autolink disabled. + EXTRA_OPTIONS=( + "--define" "library_autolink=disabled") + bazel test "${BAZEL_BUILD_OPTIONS[@]}" "${EXTRA_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" echo "bazel debug build with tests..." bazel_envoy_binary_build debug diff --git a/ci/osx-build-config/extensions_build_config.bzl b/ci/osx-build-config/extensions_build_config.bzl index 1c96ee887c44..d0ff8fdfacf9 100644 --- a/ci/osx-build-config/extensions_build_config.bzl +++ b/ci/osx-build-config/extensions_build_config.bzl @@ -17,3 +17,7 @@ WINDOWS_EXTENSIONS = {} EXTENSION_CONFIG_VISIBILITY = ["//:extension_config"] EXTENSION_PACKAGE_VISIBILITY = ["//:extension_library"] CONTRIB_EXTENSION_PACKAGE_VISIBILITY = ["//:contrib_library"] + +# As part of (https://github.com/envoyproxy/envoy-mobile/issues/175) we turned down alwayslink for envoy libraries +# This tracks libraries that should be registered as extensions. +LEGACY_ALWAYSLINK = 1 diff --git a/contrib/generic_proxy/filters/network/source/BUILD b/contrib/generic_proxy/filters/network/source/BUILD index f37fb5a22dee..dd3b53e7413a 100644 --- a/contrib/generic_proxy/filters/network/source/BUILD +++ b/contrib/generic_proxy/filters/network/source/BUILD @@ -85,4 +85,5 @@ envoy_cc_library( "//source/common/matcher:matcher_lib", "@envoy_api//contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3:pkg_cc_proto", ], + alwayslink = 1, ) diff --git a/source/common/common/BUILD b/source/common/common/BUILD index 7931c53cd6de..b9556d1fe2a6 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -8,6 +8,10 @@ load( "envoy_package", "envoy_pch_library", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -341,6 +345,7 @@ envoy_cc_library( "@envoy_api//envoy/extensions/regex_engines/v3:pkg_cc_proto", "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( diff --git a/source/common/crypto/BUILD b/source/common/crypto/BUILD index 0b6e29babbd2..1baf0903f618 100644 --- a/source/common/crypto/BUILD +++ b/source/common/crypto/BUILD @@ -28,4 +28,6 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/singleton:threadsafe_singleton", ], + # for the singleton + alwayslink = 1, ) diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index 670068dd7f22..ba0b8049115f 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -5,6 +5,10 @@ load( "envoy_package", "envoy_select_google_grpc", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -205,6 +209,7 @@ envoy_cc_library( "//source/common/config:datasource_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( diff --git a/source/common/http/match_delegate/BUILD b/source/common/http/match_delegate/BUILD index 84cbea53861b..2e1fbaa93e01 100644 --- a/source/common/http/match_delegate/BUILD +++ b/source/common/http/match_delegate/BUILD @@ -3,6 +3,10 @@ load( "envoy_cc_library", "envoy_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -21,4 +25,5 @@ envoy_cc_library( "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/common/matcher/action/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) diff --git a/source/common/http/matching/BUILD b/source/common/http/matching/BUILD index 87837d2e6ff3..4eabed5fb7d4 100644 --- a/source/common/http/matching/BUILD +++ b/source/common/http/matching/BUILD @@ -3,6 +3,10 @@ load( "envoy_cc_library", "envoy_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -27,4 +31,5 @@ envoy_cc_library( "//source/common/http:header_utility_lib", "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) diff --git a/source/common/network/BUILD b/source/common/network/BUILD index e14c45f0cda9..e76f6fca90af 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -3,6 +3,10 @@ load( "envoy_cc_library", "envoy_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -157,6 +161,7 @@ envoy_cc_library( "//envoy/network:transport_socket_interface", "//envoy/registry", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( @@ -228,6 +233,7 @@ envoy_cc_library( "//envoy/registry", "//envoy/server:bootstrap_extension_config_interface", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( @@ -254,6 +260,7 @@ envoy_cc_library( "//source/common/event:dispatcher_includes", "@envoy_api//envoy/extensions/network/socket_interface/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( diff --git a/source/common/network/matching/BUILD b/source/common/network/matching/BUILD index ca64ff93342c..cf7033e46745 100644 --- a/source/common/network/matching/BUILD +++ b/source/common/network/matching/BUILD @@ -3,6 +3,10 @@ load( "envoy_cc_library", "envoy_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -28,4 +32,5 @@ envoy_cc_library( "//source/common/network:utility_lib", "@envoy_api//envoy/extensions/matching/common_inputs/network/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) diff --git a/source/common/quic/BUILD b/source/common/quic/BUILD index d8fd4460dbb2..990f4bbc74cf 100644 --- a/source/common/quic/BUILD +++ b/source/common/quic/BUILD @@ -3,6 +3,10 @@ load( "envoy_cc_library", "envoy_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -431,6 +435,7 @@ envoy_cc_library( "@com_github_google_quiche//:quic_core_crypto_crypto_handshake_lib", "@envoy_api//envoy/extensions/transport_sockets/quic/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) # Create a single target that contains all the libraries that register factories. diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 9e848686d8c5..5160a42a4f50 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -3,6 +3,10 @@ load( "envoy_cc_library", "envoy_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -78,6 +82,7 @@ envoy_cc_library( "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", "@envoy_api//envoy/type/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( @@ -362,6 +367,7 @@ envoy_cc_library( "//source/extensions/filters/http/common:factory_base_lib", "@envoy_api//envoy/extensions/filters/http/upstream_codec/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( diff --git a/source/common/ssl/matching/BUILD b/source/common/ssl/matching/BUILD index 4c7f7e758317..25de1502208d 100644 --- a/source/common/ssl/matching/BUILD +++ b/source/common/ssl/matching/BUILD @@ -3,6 +3,10 @@ load( "envoy_cc_library", "envoy_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -18,4 +22,5 @@ envoy_cc_library( "//envoy/network:filter_interface", "@envoy_api//envoy/extensions/matching/common_inputs/ssl/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 50206adf0256..117589ffd01d 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -4,6 +4,10 @@ load( "envoy_package", "envoy_select_enable_http3", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -357,6 +361,7 @@ envoy_cc_library( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( @@ -377,6 +382,7 @@ envoy_cc_library( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( @@ -480,6 +486,7 @@ envoy_cc_library( "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( @@ -594,6 +601,7 @@ envoy_cc_library( "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( @@ -606,6 +614,7 @@ envoy_cc_library( "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_library( diff --git a/source/common/watchdog/BUILD b/source/common/watchdog/BUILD index ce5210c0c245..2481ac8dab53 100644 --- a/source/common/watchdog/BUILD +++ b/source/common/watchdog/BUILD @@ -3,6 +3,10 @@ load( "envoy_cc_library", "envoy_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -35,4 +39,5 @@ envoy_cc_library( "//source/common/protobuf:message_validator_lib", "@envoy_api//envoy/watchdog/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) diff --git a/source/extensions/access_loggers/wasm/BUILD b/source/extensions/access_loggers/wasm/BUILD index 3ea6a5ba7c0f..5765746237c4 100644 --- a/source/extensions/access_loggers/wasm/BUILD +++ b/source/extensions/access_loggers/wasm/BUILD @@ -19,6 +19,7 @@ envoy_cc_library( "//source/common/http:header_map_lib", "//source/extensions/common/wasm:wasm_lib", ], + alwayslink = 1, ) envoy_cc_extension( diff --git a/source/extensions/bootstrap/internal_listener/BUILD b/source/extensions/bootstrap/internal_listener/BUILD index cb2db85312f7..8eff9ec555eb 100644 --- a/source/extensions/bootstrap/internal_listener/BUILD +++ b/source/extensions/bootstrap/internal_listener/BUILD @@ -19,7 +19,7 @@ envoy_cc_extension( ], ) -envoy_cc_library( +envoy_cc_extension( name = "internal_listener_registry", srcs = [ "internal_listener_registry.cc", @@ -33,9 +33,10 @@ envoy_cc_library( "//source/server:listener_manager_lib", "@envoy_api//envoy/extensions/bootstrap/internal_listener/v3:pkg_cc_proto", ], + alwayslink = 1, ) -envoy_cc_library( +envoy_cc_extension( name = "client_connection_factory", srcs = [ "client_connection_factory.cc", diff --git a/source/extensions/common/wasm/BUILD b/source/extensions/common/wasm/BUILD index 5cc45cd98cc7..0b9ad7e97628 100644 --- a/source/extensions/common/wasm/BUILD +++ b/source/extensions/common/wasm/BUILD @@ -1,5 +1,6 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", "envoy_cc_library", "envoy_extension_package", ) @@ -17,6 +18,7 @@ envoy_cc_library( "//envoy/config:typed_config_interface", "@proxy_wasm_cpp_host//:wasm_vm_headers", ], + alwayslink = 1, ) # NB: Used to break the circular dependency between wasm_lib and null_plugin_lib. @@ -49,9 +51,10 @@ envoy_cc_library( "@proxy_wasm_cpp_host//:headers", "@proxy_wasm_cpp_sdk//:common_lib", ], + alwayslink = 1, ) -envoy_cc_library( +envoy_cc_extension( name = "wasm_lib", srcs = [ "context.cc", @@ -109,4 +112,5 @@ envoy_cc_library( ], }, ), + alwayslink = 1, ) diff --git a/source/extensions/common/wasm/ext/BUILD b/source/extensions/common/wasm/ext/BUILD index 6363ae9404ca..12903ef33fe5 100644 --- a/source/extensions/common/wasm/ext/BUILD +++ b/source/extensions/common/wasm/ext/BUILD @@ -5,6 +5,11 @@ load( "envoy_extension_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) + licenses(["notice"]) # Apache 2 envoy_extension_package() @@ -19,6 +24,7 @@ envoy_cc_library( "@proxy_wasm_cpp_sdk//:api_lib", "@proxy_wasm_cpp_sdk//:common_lib", ], + alwayslink = 1, ) envoy_cc_library( @@ -32,6 +38,7 @@ envoy_cc_library( "//source/common/grpc:async_client_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], + alwayslink = 1, ) # NB: this target is compiled to Wasm. Hence the generic rule. diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index f97b97e7a4f6..2169d2eb42f7 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -376,3 +376,7 @@ EXTENSIONS = { EXTENSION_CONFIG_VISIBILITY = ["//:extension_config", "//:contrib_library", "//:examples_library"] EXTENSION_PACKAGE_VISIBILITY = ["//:extension_library", "//:contrib_library", "//:examples_library"] CONTRIB_EXTENSION_PACKAGE_VISIBILITY = ["//:contrib_library"] + +# Set this variable to true to disable alwayslink for envoy_cc_library. +# TODO(alyssawilk) audit uses of this in source/ and migrate all libraries to extensions. +LEGACY_ALWAYSLINK = 1 diff --git a/source/extensions/filters/http/wasm/BUILD b/source/extensions/filters/http/wasm/BUILD index 4b5bb15995a3..b9eb2641e9ab 100644 --- a/source/extensions/filters/http/wasm/BUILD +++ b/source/extensions/filters/http/wasm/BUILD @@ -22,6 +22,7 @@ envoy_cc_library( "//source/extensions/common/wasm:wasm_lib", "@envoy_api//envoy/extensions/filters/http/wasm/v3:pkg_cc_proto", ], + alwayslink = 1, ) envoy_cc_extension( diff --git a/source/extensions/filters/network/dubbo_proxy/BUILD b/source/extensions/filters/network/dubbo_proxy/BUILD index 71bffff76c7e..0f91338b37fd 100644 --- a/source/extensions/filters/network/dubbo_proxy/BUILD +++ b/source/extensions/filters/network/dubbo_proxy/BUILD @@ -46,6 +46,7 @@ envoy_cc_library( "//envoy/buffer:buffer_interface", "//source/common/singleton:const_singleton", ], + alwayslink = 1, ) envoy_cc_library( @@ -77,6 +78,7 @@ envoy_cc_library( ":serializer_interface", "//source/common/singleton:const_singleton", ], + alwayslink = 1, ) envoy_cc_library( diff --git a/source/extensions/filters/network/dubbo_proxy/router/BUILD b/source/extensions/filters/network/dubbo_proxy/router/BUILD index e92f5d740c0d..f75fc1d75d62 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/BUILD +++ b/source/extensions/filters/network/dubbo_proxy/router/BUILD @@ -49,6 +49,7 @@ envoy_cc_library( "//source/extensions/filters/network/dubbo_proxy/filters:filter_config_interface", "@envoy_api//envoy/extensions/filters/network/dubbo_proxy/router/v3:pkg_cc_proto", ], + alwayslink = 1, ) envoy_cc_library( diff --git a/source/extensions/filters/network/thrift_proxy/BUILD b/source/extensions/filters/network/thrift_proxy/BUILD index a8160b82c59f..3ec89e0d86bd 100644 --- a/source/extensions/filters/network/thrift_proxy/BUILD +++ b/source/extensions/filters/network/thrift_proxy/BUILD @@ -206,6 +206,7 @@ envoy_cc_library( ":twitter_protocol_lib", "//source/common/common:macros", ], + alwayslink = 1, ) envoy_cc_library( @@ -221,6 +222,7 @@ envoy_cc_library( ":protocol_interface", "//source/common/common:macros", ], + alwayslink = 1, ) envoy_cc_library( @@ -237,6 +239,7 @@ envoy_cc_library( ":protocol_interface", "//source/common/common:macros", ], + alwayslink = 1, ) envoy_cc_library( @@ -254,6 +257,7 @@ envoy_cc_library( ":thrift_object_lib", "//source/common/common:macros", ], + alwayslink = 1, ) envoy_cc_library( @@ -351,6 +355,7 @@ envoy_cc_library( ":unframed_transport_lib", "//source/common/common:assert_lib", ], + alwayslink = 1, ) envoy_cc_library( @@ -366,6 +371,7 @@ envoy_cc_library( ":transport_interface", "//source/common/common:assert_lib", ], + alwayslink = 1, ) envoy_cc_library( @@ -382,6 +388,7 @@ envoy_cc_library( ":transport_interface", "//source/common/common:assert_lib", ], + alwayslink = 1, ) envoy_cc_library( @@ -397,4 +404,5 @@ envoy_cc_library( ":transport_interface", "//source/common/common:assert_lib", ], + alwayslink = 1, ) diff --git a/source/extensions/filters/network/wasm/BUILD b/source/extensions/filters/network/wasm/BUILD index 8b3bb8ac01fb..c3cee216c2e9 100644 --- a/source/extensions/filters/network/wasm/BUILD +++ b/source/extensions/filters/network/wasm/BUILD @@ -22,6 +22,7 @@ envoy_cc_library( "//source/extensions/filters/network:well_known_names", "@envoy_api//envoy/extensions/filters/network/wasm/v3:pkg_cc_proto", ], + alwayslink = 1, ) envoy_cc_extension( diff --git a/source/extensions/grpc_credentials/example/BUILD b/source/extensions/grpc_credentials/example/BUILD index 01c0e4e8ea23..e48338fe36fb 100644 --- a/source/extensions/grpc_credentials/example/BUILD +++ b/source/extensions/grpc_credentials/example/BUILD @@ -28,4 +28,5 @@ envoy_cc_library( "//source/common/grpc:google_grpc_creds_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], + alwayslink = 1, ) diff --git a/source/extensions/quic/connection_id_generator/BUILD b/source/extensions/quic/connection_id_generator/BUILD index 4f1013c84a77..d379daeaea3b 100644 --- a/source/extensions/quic/connection_id_generator/BUILD +++ b/source/extensions/quic/connection_id_generator/BUILD @@ -4,6 +4,10 @@ load( "envoy_cc_library", "envoy_extension_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -37,6 +41,7 @@ envoy_cc_library( "//source/common/quic:envoy_quic_connection_id_generator_factory_interface", "@envoy_api//envoy/extensions/quic/connection_id_generator/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_extension( diff --git a/source/extensions/quic/crypto_stream/BUILD b/source/extensions/quic/crypto_stream/BUILD index 60db832707b4..3795489dfcc6 100644 --- a/source/extensions/quic/crypto_stream/BUILD +++ b/source/extensions/quic/crypto_stream/BUILD @@ -4,6 +4,10 @@ load( "envoy_cc_library", "envoy_extension_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -25,6 +29,7 @@ envoy_cc_library( "//source/common/quic:envoy_quic_crypto_stream_factory_lib", "@envoy_api//envoy/extensions/quic/crypto_stream/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_extension( @@ -57,4 +62,5 @@ envoy_cc_library( deps = [ "//source/common/quic:envoy_quic_crypto_stream_factory_lib", ], + alwayslink = LEGACY_ALWAYSLINK, ) diff --git a/source/extensions/quic/proof_source/BUILD b/source/extensions/quic/proof_source/BUILD index 87dfd8b6023b..f2d51ed0089a 100644 --- a/source/extensions/quic/proof_source/BUILD +++ b/source/extensions/quic/proof_source/BUILD @@ -4,6 +4,10 @@ load( "envoy_cc_library", "envoy_extension_package", ) +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) licenses(["notice"]) # Apache 2 @@ -25,6 +29,7 @@ envoy_cc_library( "//source/common/quic:envoy_quic_proof_source_lib", "@envoy_api//envoy/extensions/quic/proof_source/v3:pkg_cc_proto", ], + alwayslink = LEGACY_ALWAYSLINK, ) envoy_cc_extension( diff --git a/source/extensions/stat_sinks/wasm/BUILD b/source/extensions/stat_sinks/wasm/BUILD index ce5f63bfb616..71c6a09f50ef 100644 --- a/source/extensions/stat_sinks/wasm/BUILD +++ b/source/extensions/stat_sinks/wasm/BUILD @@ -24,6 +24,7 @@ envoy_cc_extension( "//source/server:configuration_lib", "@envoy_api//envoy/extensions/stat_sinks/wasm/v3:pkg_cc_proto", ], + alwayslink = 1, ) envoy_cc_library( @@ -34,4 +35,5 @@ envoy_cc_library( "//source/extensions/common/wasm:wasm_lib", "@envoy_api//envoy/extensions/filters/network/wasm/v3:pkg_cc_proto", ], + alwayslink = 1, ) diff --git a/source/extensions/transport_sockets/tls/cert_validator/BUILD b/source/extensions/transport_sockets/tls/cert_validator/BUILD index 9b827c538104..c79416930672 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/BUILD +++ b/source/extensions/transport_sockets/tls/cert_validator/BUILD @@ -47,4 +47,5 @@ envoy_cc_library( "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", ], + alwayslink = 1, ) diff --git a/test/extensions/access_loggers/wasm/test_data/BUILD b/test/extensions/access_loggers/wasm/test_data/BUILD index 7dbdc09c5d1e..d9b33eead08d 100644 --- a/test/extensions/access_loggers/wasm/test_data/BUILD +++ b/test/extensions/access_loggers/wasm/test_data/BUILD @@ -1,15 +1,20 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_cc_library", + "envoy_cc_test_library", "envoy_package", ) + +load( + "@envoy_build_config//:extensions_build_config.bzl", + "LEGACY_ALWAYSLINK", +) load("//bazel/wasm:wasm.bzl", "envoy_wasm_cc_binary") licenses(["notice"]) # Apache 2 envoy_package() -envoy_cc_library( +envoy_cc_test_library( name = "test_cpp_plugin", srcs = [ "test_cpp.cc", diff --git a/test/extensions/bootstrap/wasm/test_data/BUILD b/test/extensions/bootstrap/wasm/test_data/BUILD index 7caf434c4771..e5fca0841f8e 100644 --- a/test/extensions/bootstrap/wasm/test_data/BUILD +++ b/test/extensions/bootstrap/wasm/test_data/BUILD @@ -1,6 +1,6 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_cc_library", + "envoy_cc_test_library", "envoy_package", ) load("//bazel/wasm:wasm.bzl", "envoy_wasm_cc_binary", "wasm_rust_binary") @@ -19,7 +19,7 @@ wasm_rust_binary( ], ) -envoy_cc_library( +envoy_cc_test_library( name = "speed_cpp_plugin", srcs = [ "speed_cpp.cc", @@ -35,7 +35,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_cc_test_library( name = "start_cpp_plugin", srcs = [ "start_cpp.cc", @@ -51,7 +51,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_cc_test_library( name = "stats_cpp_plugin", srcs = [ "stats_cpp.cc", diff --git a/test/extensions/common/wasm/test_data/BUILD b/test/extensions/common/wasm/test_data/BUILD index 35af4271860d..4a3760381f05 100644 --- a/test/extensions/common/wasm/test_data/BUILD +++ b/test/extensions/common/wasm/test_data/BUILD @@ -1,6 +1,6 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_cc_library", + "envoy_cc_test_library", "envoy_package", ) load("//bazel:envoy_select.bzl", "envoy_select_wasm_v8_bool") @@ -17,7 +17,7 @@ wasm_rust_binary( precompile = envoy_select_wasm_v8_bool(), ) -envoy_cc_library( +envoy_cc_test_library( name = "test_cpp_plugin", srcs = [ "test_cpp.cc", @@ -33,7 +33,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_cc_test_library( name = "test_context_cpp_plugin", srcs = [ "test_context_cpp.cc", @@ -50,7 +50,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_cc_test_library( name = "test_restriction_cpp_plugin", srcs = [ "test_restriction_cpp.cc", diff --git a/test/extensions/filters/http/wasm/test_data/BUILD b/test/extensions/filters/http/wasm/test_data/BUILD index c6d1f956a59e..ff6c6b137c35 100644 --- a/test/extensions/filters/http/wasm/test_data/BUILD +++ b/test/extensions/filters/http/wasm/test_data/BUILD @@ -1,7 +1,7 @@ load("@rules_proto//proto:defs.bzl", "proto_library") load( "//bazel:envoy_build_system.bzl", - "envoy_cc_library", + "envoy_cc_test_library", "envoy_package", ) load("//bazel/wasm:wasm.bzl", "envoy_wasm_cc_binary", "wasm_rust_binary") @@ -112,7 +112,7 @@ wasm_rust_binary( ], ) -envoy_cc_library( +envoy_cc_test_library( name = "test_cpp_plugin", srcs = [ "test_async_call_cpp.cc", diff --git a/test/extensions/filters/network/wasm/test_data/BUILD b/test/extensions/filters/network/wasm/test_data/BUILD index 5281609f62ae..ba975cbba256 100644 --- a/test/extensions/filters/network/wasm/test_data/BUILD +++ b/test/extensions/filters/network/wasm/test_data/BUILD @@ -1,6 +1,6 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_cc_library", + "envoy_cc_test_library", "envoy_package", ) load("//bazel/wasm:wasm.bzl", "envoy_wasm_cc_binary", "wasm_rust_binary") @@ -45,7 +45,7 @@ wasm_rust_binary( ], ) -envoy_cc_library( +envoy_cc_test_library( name = "test_cpp_plugin", srcs = [ "test_close_stream_cpp.cc", diff --git a/test/extensions/stats_sinks/wasm/BUILD b/test/extensions/stats_sinks/wasm/BUILD index 06de37b40f37..8f40393ed055 100644 --- a/test/extensions/stats_sinks/wasm/BUILD +++ b/test/extensions/stats_sinks/wasm/BUILD @@ -42,6 +42,7 @@ envoy_extension_cc_test( deps = [ "//source/common/stats:stats_lib", "//source/extensions/common/wasm:wasm_lib", + "//source/extensions/wasm_runtime/null:config", "//test/extensions/common/wasm:wasm_runtime", "//test/extensions/stats_sinks/wasm/test_data:test_context_cpp_plugin", "//test/mocks/stats:stats_mocks", diff --git a/test/extensions/stats_sinks/wasm/test_data/BUILD b/test/extensions/stats_sinks/wasm/test_data/BUILD index f2dca30447bc..ac1a12204812 100644 --- a/test/extensions/stats_sinks/wasm/test_data/BUILD +++ b/test/extensions/stats_sinks/wasm/test_data/BUILD @@ -1,6 +1,6 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_cc_library", + "envoy_cc_test_library", "envoy_package", ) load("//bazel/wasm:wasm.bzl", "envoy_wasm_cc_binary") @@ -9,7 +9,7 @@ licenses(["notice"]) # Apache 2 envoy_package() -envoy_cc_library( +envoy_cc_test_library( name = "test_context_cpp_plugin", srcs = [ "test_context_cpp.cc", From 3ea63d73407f5af8992e20e1bf0fb4b481b71d13 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Wed, 2 Nov 2022 00:44:00 +0700 Subject: [PATCH 066/112] tools, github: Add error handling and update README (#23772) * tools, github: Add error handling and update README Signed-off-by: Dhi Aurrahman --- bazel/README.md | 13 +++++++++++- tools/github/write_current_source_version.py | 22 +++++++++++++++----- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/bazel/README.md b/bazel/README.md index 2d238c7cc34d..468e22e390d6 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -41,9 +41,20 @@ Given all required [Envoy dependencies](https://www.envoyproxy.io/docs/envoy/lat 1. `python3 tools/github/write_current_source_version.py` from the repository root. 1. `bazel build -c opt envoy` from the repository root. -> Note: If the the `write_current_source_version.py` script is missing from the extracted source code directory, you can download it from [here](https://raw.githubusercontent.com/envoyproxy/envoy/main/tools/github/write_current_source_version.py). +> **Note**: If the the `write_current_source_version.py` script is missing from the extracted source code directory, you can download it from [here](https://raw.githubusercontent.com/envoyproxy/envoy/main/tools/github/write_current_source_version.py). > This script is used to generate SOURCE_VERSION that is required by [`bazel/get_workspace_status`](./get_workspace_status) to "stamp" the binary in a non-git directory. +> **Note**: To avoid rate-limiting by GitHub API, you can provide [a valid GitHub token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats) to `GITHUB_TOKEN` environment variable. +> The environment variable name that holds the token can also be customized by setting `--github_api_token_env_name`. +> In a GitHub Actions workflow file, you can set this token from [`secrets.GITHUB_TOKEN`](https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret). + +Examples: + +```console +GITHUB_TOKEN= python3 tools/github/write_current_source_version.py +MY_TOKEN= python3 tools/github/write_current_source_version.py --github_api_token_env_name=MY_TOKEN +``` + ## Quick start Bazel build for developers This section describes how to and what dependencies to install to get started building Envoy with Bazel. diff --git a/tools/github/write_current_source_version.py b/tools/github/write_current_source_version.py index 684245b98c6f..1b366504743b 100644 --- a/tools/github/write_current_source_version.py +++ b/tools/github/write_current_source_version.py @@ -9,10 +9,12 @@ # tarball. import argparse +import http import json import os import pathlib import sys +import urllib.error import urllib.request if __name__ == "__main__": @@ -74,8 +76,18 @@ # To avoid rate-limited API calls. commit_info_request.add_header( "Authorization", f"{authorization_header_prefix} {github_token}") - with urllib.request.urlopen(commit_info_request) as response: - commit_info = json.loads(response.read()) - source_version_file = project_root_dir.joinpath("SOURCE_VERSION") - # Write the extracted current version commit hash "sha" to SOURCE_VERSION. - source_version_file.write_text(commit_info["sha"]) + try: + with urllib.request.urlopen(commit_info_request) as response: + commit_info = json.loads(response.read()) + source_version_file = project_root_dir.joinpath("SOURCE_VERSION") + # Write the extracted current version commit hash "sha" to SOURCE_VERSION. + source_version_file.write_text(commit_info["sha"]) + except urllib.error.HTTPError as e: + status_code = e.code + if e.code in (http.HTTPStatus.UNAUTHORIZED, http.HTTPStatus.FORBIDDEN): + print( + f"Please check the GitHub token provided in {args.github_api_token_env_name} environment variable. {e.reason}." + ) + sys.exit(1) + else: + raise Exception(f"Failed since {e.reason} ({status_code}).") From 33fce6bd1925a5c26eb41ed584de0c9422b5e4fb Mon Sep 17 00:00:00 2001 From: GiantCroc Date: Wed, 2 Nov 2022 01:59:58 +0800 Subject: [PATCH 067/112] udp_proxy: support proxy access logging (#23242) Some stats like `no_route` and `idle_timeout` can't be printed by session access log, so we need proxy-level access logging to log global stats. Additional Description: Risk Level: Low Fixes #23241 Signed-off-by: giantcroc --- .../filters/udp/udp_proxy/v3/udp_proxy.proto | 7 +- changelogs/current.yaml | 3 + .../observability/access_log/usage.rst | 44 ++++-- .../filters/udp/udp_proxy/udp_proxy_filter.cc | 55 +++++-- .../filters/udp/udp_proxy/udp_proxy_filter.h | 31 ++-- .../udp/udp_proxy/udp_proxy_filter_test.cc | 134 +++++++++++++----- 6 files changed, 206 insertions(+), 68 deletions(-) diff --git a/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto b/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto index e5bbe7f1e0c6..c9eb7316b60e 100644 --- a/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto +++ b/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto @@ -26,7 +26,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#extension: envoy.filters.udp_listener.udp_proxy] // Configuration for the UDP proxy filter. -// [#next-free-field: 10] +// [#next-free-field: 11] message UdpProxyConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.udp.udp_proxy.v2alpha.UdpProxyConfig"; @@ -105,6 +105,9 @@ message UdpProxyConfig { // to upstream host selected on first chunk receival for that "session" (identified by source IP/port and local IP/port). bool use_per_packet_load_balancing = 7; - // Configuration for access logs emitted by the UDP proxy. Note that certain UDP specific data is emitted as :ref:`Dynamic Metadata `. + // Configuration for session access logs emitted by the UDP proxy. Note that certain UDP specific data is emitted as :ref:`Dynamic Metadata `. repeated config.accesslog.v3.AccessLog access_log = 8; + + // Configuration for proxy access logs emitted by the UDP proxy. Note that certain UDP specific data is emitted as :ref:`Dynamic Metadata `. + repeated config.accesslog.v3.AccessLog proxy_access_log = 10; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index fec25a1a3dfd..55e829e05079 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -71,5 +71,8 @@ new_features: - area: http change: | allowing the dynamic forward proxy cluster to :ref:`allow_coalesced_connections ` for HTTP/2 and HTTP/3 connections. +- area: udp_proxy + change: | + added support for :ref:`proxy_access_log `. deprecated: diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index 5d9907cfd748..38211dbdee20 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -650,28 +650,50 @@ The following command operators are supported: UDP For :ref:`UDP Proxy `, - NAMESPACE should be always set to "udp.proxy", optional KEYs are as follows: + when NAMESPACE is set to "udp.proxy.session", optional KEYs are as follows: * ``cluster_name``: Name of the cluster. * ``bytes_sent``: Total number of downstream bytes sent to the upstream in the session. * ``bytes_received``: Total number of downstream bytes received from the upstream in the session. * ``errors_sent``: Number of errors that have occurred when sending datagrams to the upstream in the session. - * ``errors_received``: Number of errors that have occurred when receiving datagrams from the upstream in UDP proxy. - Since the receiving errors are counted in at the listener level (vs. the session), this counter is global to all sessions and may not be directly attributable to the session being logged. * ``datagrams_sent``: Number of datagrams sent to the upstream successfully in the session. * ``datagrams_received``: Number of datagrams received from the upstream successfully in the session. - Recommended access log format for UDP proxy: + Recommended session access log format for UDP proxy: + + .. code-block:: none + + [%START_TIME%] %DYNAMIC_METADATA(udp.proxy.session:cluster_name)% + %DYNAMIC_METADATA(udp.proxy.session:bytes_sent)% + %DYNAMIC_METADATA(udp.proxy.session:bytes_received)% + %DYNAMIC_METADATA(udp.proxy.session:errors_sent)% + %DYNAMIC_METADATA(udp.proxy.session:datagrams_sent)% + %DYNAMIC_METADATA(udp.proxy.session:datagrams_received)%\n + + when NAMESPACE is set to "udp.proxy.proxy", optional KEYs are as follows: + + * ``bytes_sent``: Total number of downstream bytes sent to the upstream in UDP proxy. + * ``bytes_received``: Total number of downstream bytes received from the upstream in UDP proxy. + * ``errors_sent``: Number of errors that have occurred when sending datagrams to the upstream in UDP proxy. + * ``errors_received``: Number of errors that have occurred when receiving datagrams from the upstream in UDP proxy. + * ``datagrams_sent``: Number of datagrams sent to the upstream successfully in UDP proxy. + * ``datagrams_received``: Number of datagrams received from the upstream successfully in UDP proxy. + * ``no_route``: Number of times that no upstream cluster found in UDP proxy. + * ``session_total``: Total number of sessions in UDP proxy. + * ``idle_timeout``: Number of times that sessions idle timeout occurred in UDP proxy. + + Recommended proxy access log format for UDP proxy: .. code-block:: none - [%START_TIME%] %DYNAMIC_METADATA(udp.proxy:cluster_name)% - %DYNAMIC_METADATA(udp.proxy:bytes_sent)% - %DYNAMIC_METADATA(udp.proxy:bytes_received)% - %DYNAMIC_METADATA(udp.proxy:errors_sent)% - %DYNAMIC_METADATA(udp.proxy:errors_received)% - %DYNAMIC_METADATA(udp.proxy:datagrams_sent)% - %DYNAMIC_METADATA(udp.proxy:datagrams_received)%\n + [%START_TIME%] + %DYNAMIC_METADATA(udp.proxy.proxy:bytes_sent)% + %DYNAMIC_METADATA(udp.proxy.proxy:bytes_received)% + %DYNAMIC_METADATA(udp.proxy.proxy:errors_sent)% + %DYNAMIC_METADATA(udp.proxy.proxy:errors_received)% + %DYNAMIC_METADATA(udp.proxy.proxy:datagrams_sent)% + %DYNAMIC_METADATA(udp.proxy.proxy:datagrams_received)% + %DYNAMIC_METADATA(udp.proxy.proxy:session_total)%\n THRIFT For :ref:`Thrift Proxy `, diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc index db9db3a84141..118d206bc786 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc @@ -20,6 +20,19 @@ UdpProxyFilter::UdpProxyFilter(Network::UdpReadFilterCallbacks& callbacks, onClusterAddOrUpdate(*cluster); } } + + if (!config_->proxyAccessLogs().empty()) { + udp_proxy_stats_.emplace(StreamInfo::StreamInfoImpl(config_->timeSource(), nullptr)); + } +} + +UdpProxyFilter::~UdpProxyFilter() { + if (!config_->proxyAccessLogs().empty()) { + fillProxyStreamInfo(); + for (const auto& access_log : config_->proxyAccessLogs()) { + access_log->log(nullptr, nullptr, nullptr, udp_proxy_stats_.value()); + } + } } void UdpProxyFilter::onClusterAddOrUpdate(Upstream::ThreadLocalCluster& cluster) { @@ -234,8 +247,8 @@ UdpProxyFilter::ActiveSession::ActiveSession(ClusterInfo& cluster, // NOTE: The socket call can only fail due to memory/fd exhaustion. No local ephemeral port // is bound until the first packet is sent to the upstream host. socket_(cluster.filter_.createSocket(host)) { - if (!cluster_.filter_.config_->accessLogs().empty()) { - udp_sess_stats_.emplace( + if (!cluster_.filter_.config_->sessionAccessLogs().empty()) { + udp_session_stats_.emplace( StreamInfo::StreamInfoImpl(cluster_.filter_.config_->timeSource(), nullptr)); } @@ -281,29 +294,51 @@ UdpProxyFilter::ActiveSession::~ActiveSession() { .connections() .dec(); - if (!cluster_.filter_.config_->accessLogs().empty()) { - fillStreamInfo(); - for (const auto& access_log : cluster_.filter_.config_->accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, udp_sess_stats_.value()); + if (!cluster_.filter_.config_->sessionAccessLogs().empty()) { + fillSessionStreamInfo(); + for (const auto& access_log : cluster_.filter_.config_->sessionAccessLogs()) { + access_log->log(nullptr, nullptr, nullptr, udp_session_stats_.value()); } } } -void UdpProxyFilter::ActiveSession::fillStreamInfo() { +void UdpProxyFilter::ActiveSession::fillSessionStreamInfo() { ProtobufWkt::Struct stats_obj; auto& fields_map = *stats_obj.mutable_fields(); fields_map["cluster_name"] = ValueUtil::stringValue(cluster_.cluster_.info()->name()); fields_map["bytes_sent"] = ValueUtil::numberValue(session_stats_.downstream_sess_tx_bytes_); fields_map["bytes_received"] = ValueUtil::numberValue(session_stats_.downstream_sess_rx_bytes_); fields_map["errors_sent"] = ValueUtil::numberValue(session_stats_.downstream_sess_tx_errors_); - fields_map["errors_received"] = - ValueUtil::numberValue(cluster_.filter_.config_->stats().downstream_sess_rx_errors_.value()); fields_map["datagrams_sent"] = ValueUtil::numberValue(session_stats_.downstream_sess_tx_datagrams_); fields_map["datagrams_received"] = ValueUtil::numberValue(session_stats_.downstream_sess_rx_datagrams_); - udp_sess_stats_.value().setDynamicMetadata("udp.proxy", stats_obj); + udp_session_stats_.value().setDynamicMetadata("udp.proxy.session", stats_obj); +} + +void UdpProxyFilter::fillProxyStreamInfo() { + ProtobufWkt::Struct stats_obj; + auto& fields_map = *stats_obj.mutable_fields(); + fields_map["bytes_sent"] = + ValueUtil::numberValue(config_->stats().downstream_sess_tx_bytes_.value()); + fields_map["bytes_received"] = + ValueUtil::numberValue(config_->stats().downstream_sess_rx_bytes_.value()); + fields_map["errors_sent"] = + ValueUtil::numberValue(config_->stats().downstream_sess_tx_errors_.value()); + fields_map["errors_received"] = + ValueUtil::numberValue(config_->stats().downstream_sess_rx_errors_.value()); + fields_map["datagrams_sent"] = + ValueUtil::numberValue(config_->stats().downstream_sess_tx_datagrams_.value()); + fields_map["datagrams_received"] = + ValueUtil::numberValue(config_->stats().downstream_sess_rx_datagrams_.value()); + fields_map["no_route"] = + ValueUtil::numberValue(config_->stats().downstream_sess_no_route_.value()); + fields_map["session_total"] = + ValueUtil::numberValue(config_->stats().downstream_sess_total_.value()); + fields_map["idle_timeout"] = ValueUtil::numberValue(config_->stats().idle_timeout_.value()); + + udp_proxy_stats_.value().setDynamicMetadata("udp.proxy.proxy", stats_obj); } void UdpProxyFilter::ActiveSession::onIdleTimer() { diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h index 77049babe030..e44edf9c5492 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h @@ -25,8 +25,6 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" -// TODO(mattklein123): UDP session access logging. - namespace Envoy { namespace Extensions { namespace UdpFilters { @@ -90,9 +88,15 @@ class UdpProxyFilterConfig { "is not running with the CAP_NET_ADMIN capability."); } - access_logs_.reserve(config.access_log_size()); + session_access_logs_.reserve(config.access_log_size()); for (const envoy::config::accesslog::v3::AccessLog& log_config : config.access_log()) { - access_logs_.emplace_back(AccessLog::AccessLogFactory::fromProto(log_config, context)); + session_access_logs_.emplace_back( + AccessLog::AccessLogFactory::fromProto(log_config, context)); + } + + proxy_access_logs_.reserve(config.proxy_access_log_size()); + for (const envoy::config::accesslog::v3::AccessLog& log_config : config.proxy_access_log()) { + proxy_access_logs_.emplace_back(AccessLog::AccessLogFactory::fromProto(log_config, context)); } if (!config.hash_policies().empty()) { @@ -116,7 +120,12 @@ class UdpProxyFilterConfig { const Network::ResolvedUdpSocketConfig& upstreamSocketConfig() const { return upstream_socket_config_; } - const std::vector& accessLogs() const { return access_logs_; } + const std::vector& sessionAccessLogs() const { + return session_access_logs_; + } + const std::vector& proxyAccessLogs() const { + return proxy_access_logs_; + } private: static UdpProxyDownstreamStats generateStats(const std::string& stat_prefix, @@ -135,7 +144,8 @@ class UdpProxyFilterConfig { std::unique_ptr hash_policy_; mutable UdpProxyDownstreamStats stats_; const Network::ResolvedUdpSocketConfig upstream_socket_config_; - std::vector access_logs_; + std::vector session_access_logs_; + std::vector proxy_access_logs_; Random::RandomGenerator& random_; }; @@ -165,6 +175,7 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, public: UdpProxyFilter(Network::UdpReadFilterCallbacks& callbacks, const UdpProxyFilterConfigSharedPtr& config); + ~UdpProxyFilter() override; // Network::UdpListenerReadFilter Network::FilterStatus onData(Network::UdpRecvData& data) override; @@ -193,7 +204,7 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, private: void onIdleTimer(); void onReadReady(); - void fillStreamInfo(); + void fillSessionStreamInfo(); // Network::UdpPacketProcessor void processPacket(Network::Address::InstanceConstSharedPtr local_address, @@ -242,7 +253,7 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, bool skip_connect_{}; UdpProxySessionStats session_stats_{}; - absl::optional udp_sess_stats_; + absl::optional udp_session_stats_; }; using ActiveSessionPtr = std::unique_ptr; @@ -380,6 +391,8 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, nullptr, Network::SocketCreationOptions{}); } + void fillProxyStreamInfo(); + // Upstream::ClusterUpdateCallbacks void onClusterAddOrUpdate(Upstream::ThreadLocalCluster& cluster) final; void onClusterRemoval(const std::string& cluster_name) override; @@ -388,6 +401,8 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, const Upstream::ClusterUpdateCallbacksHandlePtr cluster_update_callbacks_; // Map for looking up cluster info with its name. absl::flat_hash_map cluster_infos_; + + absl::optional udp_proxy_stats_; }; } // namespace UdpProxy diff --git a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc index ab418e711b65..6fae57176a9b 100644 --- a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc +++ b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc @@ -182,7 +182,8 @@ class UdpProxyFilterTest : public testing::Test { peer_address_(std::move(peer_address)) { // Disable strict mock warnings. ON_CALL(*factory_context_.access_log_manager_.file_, write(_)) - .WillByDefault(SaveArg<0>(&access_log_data_)); + .WillByDefault( + Invoke([&](absl::string_view data) { output_.push_back(std::string(data)); })); ON_CALL(os_sys_calls_, supportsIpTransparent()).WillByDefault(Return(true)); EXPECT_CALL(os_sys_calls_, supportsUdpGro()).Times(AtLeast(0)).WillRepeatedly(Return(true)); EXPECT_CALL(callbacks_, udpListener()).Times(AtLeast(0)); @@ -290,16 +291,31 @@ use_original_src_ip: true // Return the config from yaml, plus one file access log with the specified format envoy::extensions::filters::udp::udp_proxy::v3::UdpProxyConfig - accessLogConfig(const std::string& yaml, const std::string& access_log_format) { + accessLogConfig(const std::string& yaml, const std::string& session_access_log_format, + const std::string& proxy_access_log_format) { auto config = readConfig(yaml); - envoy::config::accesslog::v3::AccessLog* access_log = config.mutable_access_log()->Add(); - access_log->set_name("envoy.access_loggers.file"); - envoy::extensions::access_loggers::file::v3::FileAccessLog file_access_log; - file_access_log.set_path("unused"); - file_access_log.mutable_log_format()->mutable_text_format_source()->set_inline_string( - access_log_format); - access_log->mutable_typed_config()->PackFrom(file_access_log); + if (!session_access_log_format.empty()) { + envoy::config::accesslog::v3::AccessLog* session_access_log = + config.mutable_access_log()->Add(); + session_access_log->set_name("envoy.access_loggers.file"); + envoy::extensions::access_loggers::file::v3::FileAccessLog session_file_access_log; + session_file_access_log.set_path("unused"); + session_file_access_log.mutable_log_format()->mutable_text_format_source()->set_inline_string( + session_access_log_format); + session_access_log->mutable_typed_config()->PackFrom(session_file_access_log); + } + + if (!proxy_access_log_format.empty()) { + envoy::config::accesslog::v3::AccessLog* proxy_access_log = + config.mutable_proxy_access_log()->Add(); + proxy_access_log->set_name("envoy.access_loggers.file"); + envoy::extensions::access_loggers::file::v3::FileAccessLog proxy_file_access_log; + proxy_file_access_log.set_path("unused"); + proxy_file_access_log.mutable_log_format()->mutable_text_format_source()->set_inline_string( + proxy_access_log_format); + proxy_access_log->mutable_typed_config()->PackFrom(proxy_file_access_log); + } return config; } @@ -329,6 +345,7 @@ use_original_src_ip: true std::unique_ptr filter_; std::vector test_sessions_; StringViewSaver access_log_data_; + std::vector output_; bool expect_gro_{}; const Network::Address::InstanceConstSharedPtr upstream_address_; const Network::Address::InstanceConstSharedPtr peer_address_; @@ -383,10 +400,20 @@ class UdpProxyFilterIpv4Ipv6Test : public UdpProxyFilterIpv6Test { TEST_F(UdpProxyFilterTest, BasicFlow) { InSequence s; - const std::string access_log_format = "%DYNAMIC_METADATA(udp.proxy:bytes_received)% " - "%DYNAMIC_METADATA(udp.proxy:datagrams_received)% " - "%DYNAMIC_METADATA(udp.proxy:bytes_sent)% " - "%DYNAMIC_METADATA(udp.proxy:datagrams_sent)%"; + const std::string session_access_log_format = + "%DYNAMIC_METADATA(udp.proxy.session:bytes_received)% " + "%DYNAMIC_METADATA(udp.proxy.session:datagrams_received)% " + "%DYNAMIC_METADATA(udp.proxy.session:bytes_sent)% " + "%DYNAMIC_METADATA(udp.proxy.session:datagrams_sent)%"; + + const std::string proxy_access_log_format = + "%DYNAMIC_METADATA(udp.proxy.proxy:bytes_received)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:datagrams_received)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:bytes_sent)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:datagrams_sent)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:no_route)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:session_total)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:idle_timeout)%"; setup(accessLogConfig(R"EOF( stat_prefix: foo @@ -400,7 +427,7 @@ stat_prefix: foo upstream_socket_config: prefer_gro: false )EOF", - access_log_format), + session_access_log_format, proxy_access_log_format), true, false); expectSessionCreate(upstream_address_); @@ -425,7 +452,9 @@ stat_prefix: foo checkTransferStats(17 /*rx_bytes*/, 3 /*rx_datagrams*/, 17 /*tx_bytes*/, 3 /*tx_datagrams*/); filter_.reset(); - EXPECT_EQ(access_log_data_.value(), "17 3 17 3"); + EXPECT_EQ(output_.size(), 2); + EXPECT_EQ(output_.front(), "17 3 17 3 0 1 0"); + EXPECT_EQ(output_.back(), "17 3 17 3"); } // Route with source IP. @@ -475,7 +504,12 @@ stat_prefix: foo TEST_F(UdpProxyFilterTest, IdleTimeout) { InSequence s; - setup(readConfig(R"EOF( + const std::string session_access_log_format = ""; + + const std::string proxy_access_log_format = "%DYNAMIC_METADATA(udp.proxy.proxy:session_total)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:idle_timeout)%"; + + setup(accessLogConfig(R"EOF( stat_prefix: foo matcher: on_no_match: @@ -484,7 +518,8 @@ stat_prefix: foo typed_config: '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route cluster: fake_cluster - )EOF")); + )EOF", + session_access_log_format, proxy_access_log_format)); expectSessionCreate(upstream_address_); test_sessions_[0].expectWriteToUpstream("hello", 0, nullptr, true); @@ -501,19 +536,28 @@ stat_prefix: foo recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); EXPECT_EQ(2, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); + + filter_.reset(); + EXPECT_EQ(output_.size(), 1); + EXPECT_EQ(output_.front(), "2 1"); } // Verify downstream send and receive error handling. TEST_F(UdpProxyFilterTest, SendReceiveErrorHandling) { InSequence s; - const std::string access_log_format = "%DYNAMIC_METADATA(udp.proxy:cluster_name)% " - "%DYNAMIC_METADATA(udp.proxy:bytes_sent)% " - "%DYNAMIC_METADATA(udp.proxy:bytes_received)% " - "%DYNAMIC_METADATA(udp.proxy:errors_sent)% " - "%DYNAMIC_METADATA(udp.proxy:errors_received)% " - "%DYNAMIC_METADATA(udp.proxy:datagrams_sent)% " - "%DYNAMIC_METADATA(udp.proxy:datagrams_received)%"; + const std::string session_access_log_format = + "%DYNAMIC_METADATA(udp.proxy.session:cluster_name)% " + "%DYNAMIC_METADATA(udp.proxy.session:bytes_sent)% " + "%DYNAMIC_METADATA(udp.proxy.session:bytes_received)% " + "%DYNAMIC_METADATA(udp.proxy.session:errors_sent)% " + "%DYNAMIC_METADATA(udp.proxy.session:datagrams_sent)% " + "%DYNAMIC_METADATA(udp.proxy.session:datagrams_received)%"; + + const std::string proxy_access_log_format = "%DYNAMIC_METADATA(udp.proxy.proxy:errors_received)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:no_route)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:session_total)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:idle_timeout)%"; setup(accessLogConfig(R"EOF( stat_prefix: foo @@ -525,7 +569,7 @@ stat_prefix: foo '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route cluster: fake_cluster )EOF", - access_log_format)); + session_access_log_format, proxy_access_log_format)); filter_->onReceiveError(Api::IoError::IoErrorCode::UnknownError); EXPECT_EQ(1, config_->stats().downstream_sess_rx_errors_.value()); @@ -565,20 +609,25 @@ stat_prefix: foo ->value()); filter_.reset(); - EXPECT_EQ(access_log_data_.value(), "fake_cluster 0 10 1 1 0 2"); + EXPECT_EQ(output_.size(), 2); + EXPECT_EQ(output_.front(), "1 0 1 0"); + EXPECT_EQ(output_.back(), "fake_cluster 0 10 1 0 2"); } // Verify upstream connect error handling. TEST_F(UdpProxyFilterTest, ConnectErrorHandling) { InSequence s; - const std::string access_log_format = "%DYNAMIC_METADATA(udp.proxy:cluster_name)% " - "%DYNAMIC_METADATA(udp.proxy:bytes_sent)% " - "%DYNAMIC_METADATA(udp.proxy:bytes_received)% " - "%DYNAMIC_METADATA(udp.proxy:errors_sent)% " - "%DYNAMIC_METADATA(udp.proxy:errors_received)% " - "%DYNAMIC_METADATA(udp.proxy:datagrams_sent)% " - "%DYNAMIC_METADATA(udp.proxy:datagrams_received)%"; + const std::string session_access_log_format = + "%DYNAMIC_METADATA(udp.proxy.session:cluster_name)% " + "%DYNAMIC_METADATA(udp.proxy.session:bytes_sent)% " + "%DYNAMIC_METADATA(udp.proxy.session:bytes_received)% " + "%DYNAMIC_METADATA(udp.proxy.session:errors_sent)% " + "%DYNAMIC_METADATA(udp.proxy.session:datagrams_sent)% " + "%DYNAMIC_METADATA(udp.proxy.session:datagrams_received)%"; + + const std::string proxy_access_log_format = "%DYNAMIC_METADATA(udp.proxy.proxy:errors_received)% " + "%DYNAMIC_METADATA(udp.proxy.proxy:session_total)%"; setup(accessLogConfig(R"EOF( stat_prefix: foo @@ -590,7 +639,7 @@ stat_prefix: foo '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route cluster: fake_cluster )EOF", - access_log_format)); + session_access_log_format, proxy_access_log_format)); expectSessionCreate(upstream_address_); test_sessions_[0].expectWriteToUpstream("hello", 0, nullptr, true, 1); @@ -604,7 +653,9 @@ stat_prefix: foo ->value()); filter_.reset(); - EXPECT_EQ(access_log_data_.value(), "fake_cluster 0 5 0 0 0 1"); + EXPECT_EQ(output_.size(), 2); + EXPECT_EQ(output_.front(), "0 1"); + EXPECT_EQ(output_.back(), "fake_cluster 0 5 0 0 1"); } // No upstream host handling. @@ -633,7 +684,11 @@ stat_prefix: foo TEST_F(UdpProxyFilterTest, NoUpstreamClusterAtCreation) { InSequence s; - setup(readConfig(R"EOF( + const std::string session_access_log_format = ""; + + const std::string proxy_access_log_format = "%DYNAMIC_METADATA(udp.proxy.proxy:no_route)%"; + + setup(accessLogConfig(R"EOF( stat_prefix: foo matcher: on_no_match: @@ -642,11 +697,16 @@ stat_prefix: foo typed_config: '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route cluster: fake_cluster - )EOF"), + )EOF", + session_access_log_format, proxy_access_log_format), false); recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); EXPECT_EQ(1, config_->stats().downstream_sess_no_route_.value()); + + filter_.reset(); + EXPECT_EQ(output_.size(), 1); + EXPECT_EQ(output_.front(), "1"); } // Dynamic cluster addition and removal handling. From 84df237d6576341299ab8c5cca9741e6e1f32008 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 1 Nov 2022 13:33:48 -0700 Subject: [PATCH 068/112] bazel: update to 6.0.0rc1 (#23678) Signed-off-by: Keith Smiley --- .bazelrc | 1 + .bazelversion | 2 +- bazel/repository_locations.bzl | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index eea01acb6217..723b5cd4fe00 100644 --- a/.bazelrc +++ b/.bazelrc @@ -377,6 +377,7 @@ build:windows --define tcmalloc=disabled build:windows --define wasm=disabled build:windows --define manual_stamp=manual_stamp build:windows --cxxopt="/std:c++17" +build:windows --output_groups=+pdb_file # TODO(wrowe,sunjayBhatia): Resolve bugs upstream in curl and rules_foreign_cc # See issue https://github.com/bazelbuild/rules_foreign_cc/issues/301 diff --git a/.bazelversion b/.bazelversion index 9f2e85218f69..ba8bc581b152 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.0.0-pre.20220706.4 +6.0.0rc1 diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 1e9427c12750..11701755942c 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1376,6 +1376,19 @@ REPOSITORY_LOCATIONS_SPEC = dict( release_date = "2022-03-30", cpe = "N/A", ), + rules_license = dict( + project_name = "rules_license", + project_desc = "Bazel rules for checking open source licenses", + project_url = "https://github.com/bazelbuild/rules_license", + version = "0.0.3", + sha256 = "00ccc0df21312c127ac4b12880ab0f9a26c1cff99442dc6c5a331750360de3c3", + urls = ["https://github.com/bazelbuild/rules_license/releases/download/0.0.3/rules_license-0.0.3.tar.gz"], + use_category = ["build", "dataplane_core", "controlplane"], + release_date = "2022-05-28", + cpe = "N/A", + license = "Apache-2.0", + license_url = "https://github.com/bazelbuild/rules_license/blob/{version}/LICENSE", + ), ) def _compiled_protoc_deps(locations, versions): From 551eba06a3499530babe49b93070b54deafee802 Mon Sep 17 00:00:00 2001 From: doujiang Date: Wed, 2 Nov 2022 05:10:52 +0800 Subject: [PATCH 069/112] bazel: also export symbols in test. (#23771) It makes the downstream project easier to reuse envoy_cc_test, when there are tests depends on exported symbols. Signed-off-by: doujiang24 --- bazel/envoy_binary.bzl | 26 ++++---------------------- bazel/envoy_internal.bzl | 20 ++++++++++++++++++++ bazel/envoy_test.bzl | 6 +++++- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/bazel/envoy_binary.bzl b/bazel/envoy_binary.bzl index 84b5b9458b28..b435f69659ab 100644 --- a/bazel/envoy_binary.bzl +++ b/bazel/envoy_binary.bzl @@ -4,7 +4,9 @@ load( ":envoy_internal.bzl", "envoy_copts", "envoy_dbg_linkopts", + "envoy_exported_symbols_input", "envoy_external_dep_path", + "envoy_select_exported_symbols", "envoy_stdlib_deps", "tcmalloc_external_dep", ) @@ -23,7 +25,7 @@ def envoy_cc_binary( linkopts = [], tags = [], features = []): - linker_inputs = _envoy_exported_symbols_input() + linker_inputs = envoy_exported_symbols_input() if not linkopts: linkopts = _envoy_linkopts() @@ -49,26 +51,6 @@ def envoy_cc_binary( features = features, ) -def _envoy_exported_symbols_input(): - return ["@envoy//bazel:exported_symbols.txt"] - -# Default symbols to be exported. -# TODO(wbpcode): make this work correctly for apple/darwin. -def _envoy_default_exported_symbols(): - return select({ - "@envoy//bazel:linux": [ - "-Wl,--dynamic-list=$(location @envoy//bazel:exported_symbols.txt)", - ], - "//conditions:default": [], - }) - -# Select the given values if exporting is enabled in the current build. -def _envoy_select_exported_symbols(xs): - return select({ - "@envoy//bazel:enable_exported_symbols": xs, - "//conditions:default": [], - }) + _envoy_default_exported_symbols() - # Compute the final linkopts based on various options. def _envoy_linkopts(): return select({ @@ -98,7 +80,7 @@ def _envoy_linkopts(): "@envoy//bazel:boringssl_fips": [], "@envoy//bazel:windows_x86_64": [], "//conditions:default": ["-pie"], - }) + _envoy_select_exported_symbols(["-Wl,-E"]) + }) + envoy_select_exported_symbols(["-Wl,-E"]) def _envoy_stamped_deps(): return select({ diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index 76bce7f05a71..df63d57e1568 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -197,3 +197,23 @@ def _envoy_select_perfetto(xs): "@envoy//bazel:enable_perf_tracing": xs, "//conditions:default": [], }) + +def envoy_exported_symbols_input(): + return ["@envoy//bazel:exported_symbols.txt"] + +# Default symbols to be exported. +# TODO(wbpcode): make this work correctly for apple/darwin. +def _envoy_default_exported_symbols(): + return select({ + "@envoy//bazel:linux": [ + "-Wl,--dynamic-list=$(location @envoy//bazel:exported_symbols.txt)", + ], + "//conditions:default": [], + }) + +# Select the given values if exporting is enabled in the current build. +def envoy_select_exported_symbols(xs): + return select({ + "@envoy//bazel:enable_exported_symbols": xs, + "//conditions:default": [], + }) + _envoy_default_exported_symbols() diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index fc66a619758e..70271833dec7 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -9,8 +9,10 @@ load( ":envoy_internal.bzl", "envoy_copts", "envoy_dbg_linkopts", + "envoy_exported_symbols_input", "envoy_external_dep_path", "envoy_linkstatic", + "envoy_select_exported_symbols", "envoy_select_force_libcpp", "envoy_stdlib_deps", "tcmalloc_external_dep", @@ -70,7 +72,7 @@ def _envoy_test_linkopts(): # TODO(mattklein123): It's not great that we universally link against the following libs. # In particular, -latomic and -lrt are not needed on all platforms. Make this more granular. "//conditions:default": ["-pthread", "-lrt", "-ldl"], - }) + envoy_select_force_libcpp([], ["-lstdc++fs", "-latomic"]) + envoy_dbg_linkopts() + }) + envoy_select_force_libcpp([], ["-lstdc++fs", "-latomic"]) + envoy_dbg_linkopts() + envoy_select_exported_symbols(["-Wl,-E"]) # Envoy C++ fuzz test targets. These are not included in coverage runs. def envoy_cc_fuzz_test( @@ -106,6 +108,7 @@ def envoy_cc_fuzz_test( native.cc_test( name = name, copts = envoy_copts("@envoy", test = True), + additional_linker_inputs = envoy_exported_symbols_input(), linkopts = _envoy_test_linkopts() + select({ "@envoy//bazel:libfuzzer": ["-fsanitize=fuzzer"], "//conditions:default": [], @@ -166,6 +169,7 @@ def envoy_cc_test( srcs = srcs, data = data, copts = envoy_copts(repository, test = True) + copts + envoy_pch_copts(repository, "//test:test_pch"), + additional_linker_inputs = envoy_exported_symbols_input(), linkopts = _envoy_test_linkopts(), linkstatic = envoy_linkstatic(), malloc = tcmalloc_external_dep(repository), From 851606902390f9e650002538114f89ab95596ab4 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 1 Nov 2022 15:17:29 -0700 Subject: [PATCH 070/112] bazel: workaround boringssl warning (#23777) New warning with gcc 12, upstream issue: https://bugs.chromium.org/p/boringssl/issues/detail?id=492 Fixes https://github.com/envoyproxy/envoy/issues/23568 Signed-off-by: Keith Smiley --- bazel/boringssl_static.patch | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bazel/boringssl_static.patch b/bazel/boringssl_static.patch index 51787271182f..35c77227fcb2 100644 --- a/bazel/boringssl_static.patch +++ b/bazel/boringssl_static.patch @@ -1,8 +1,17 @@ diff --git a/BUILD b/BUILD -index cfa695a44..e748f9b46 100644 +index 1ec2bdf18..50fe0f050 100644 --- a/BUILD +++ b/BUILD -@@ -101,7 +101,10 @@ linux_copts = posix_copts + [ +@@ -96,12 +96,19 @@ linux_copts = posix_copts + [ + # it should not be set on Apple platforms, where it instead disables APIs + # we use. See compat(5) and sys/cdefs.h. + "-D_XOPEN_SOURCE=700", ++ # TODO(keith): Remove https://bugs.chromium.org/p/boringssl/issues/detail?id=492 ++ "-Wno-array-bounds", ++ "-Wno-stringop-overflow", ++ "-Wno-unknown-warning-option", + ] + boringssl_copts = select({ "@platforms//os:linux": linux_copts, "@platforms//os:macos": posix_copts, @@ -14,7 +23,7 @@ index cfa695a44..e748f9b46 100644 "//conditions:default": [], }) -@@ -163,6 +166,7 @@ cc_library( +@@ -163,6 +170,7 @@ cc_library( "@platforms//os:windows": ["-defaultlib:advapi32.lib"], "//conditions:default": ["-lpthread"], }), @@ -22,7 +31,7 @@ index cfa695a44..e748f9b46 100644 visibility = ["//visibility:public"], ) -@@ -172,6 +176,7 @@ cc_library( +@@ -172,6 +180,7 @@ cc_library( hdrs = ssl_headers, copts = boringssl_copts_cxx, includes = ["src/include"], From 94b0d9e242937a01348c8f1e3aa27d1b7ee3c723 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 1 Nov 2022 18:28:53 -0700 Subject: [PATCH 071/112] bazel: fix URL version variable usage (#23780) Signed-off-by: Keith Smiley --- bazel/repository_locations.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 11701755942c..62c537da0105 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1382,7 +1382,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_url = "https://github.com/bazelbuild/rules_license", version = "0.0.3", sha256 = "00ccc0df21312c127ac4b12880ab0f9a26c1cff99442dc6c5a331750360de3c3", - urls = ["https://github.com/bazelbuild/rules_license/releases/download/0.0.3/rules_license-0.0.3.tar.gz"], + urls = ["https://github.com/bazelbuild/rules_license/releases/download/{version}/rules_license-{version}.tar.gz"], use_category = ["build", "dataplane_core", "controlplane"], release_date = "2022-05-28", cpe = "N/A", From 1c68be22da99ef840020fca5980b1905ca7c2dee Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 1 Nov 2022 18:29:46 -0700 Subject: [PATCH 072/112] bazel: update tcmalloc (#23671) Fixes https://github.com/envoyproxy/envoy/issues/23580 Full diff: https://github.com/google/tcmalloc/compare/59400332b9cff9920b6a1da203ac1575272a9f44...e33c7bc60415127c104006d3301c96902f98d42a Signed-off-by: Keith Smiley --- bazel/repository_locations.bzl | 6 +++--- test/common/stats/thread_local_store_test.cc | 4 ++-- test/integration/stats_integration_test.cc | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 62c537da0105..ae438af5749e 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -336,12 +336,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "tcmalloc", project_desc = "Fast, multi-threaded malloc implementation", project_url = "https://github.com/google/tcmalloc", - version = "59400332b9cff9920b6a1da203ac1575272a9f44", - sha256 = "3e0a0c135318fa69e748b140d32cb24a64d885bedfeb5f23fa01cc0d7859bbf0", + version = "e33c7bc60415127c104006d3301c96902f98d42a", + sha256 = "14a2c91b71d6719558768a79671408c9acd8284b418e80386c5888047e2c15aa", strip_prefix = "tcmalloc-{version}", urls = ["https://github.com/google/tcmalloc/archive/{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2022-08-06", + release_date = "2022-10-24", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/google/tcmalloc/blob/{version}/LICENSE", diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 54a4f65c1388..93e633fbe999 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -1381,7 +1381,7 @@ TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithoutTlsRealSymbolTable) { TestUtil::forEachSampleStat( 100, true, [this](absl::string_view name) { store_.counterFromString(std::string(name)); }); EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 688080); // July 2, 2020 - EXPECT_MEMORY_LE(memory_test.consumedBytes(), 0.75 * million_); + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 0.85 * million_); } TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithTlsRealSymbolTable) { @@ -1390,7 +1390,7 @@ TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithTlsRealSymbolTable) { TestUtil::forEachSampleStat( 100, true, [this](absl::string_view name) { store_.counterFromString(std::string(name)); }); EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 827616); // Sep 25, 2020 - EXPECT_MEMORY_LE(memory_test.consumedBytes(), 0.9 * million_); + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 0.97 * million_); } TEST_F(StatsThreadLocalStoreTest, ShuttingDown) { diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index 1a9ab97eb368..0ce195d2d6d8 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -378,6 +378,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSize) { // searching // 2021/08/18 13176 40577 40700 Support slow start mode // 2022/03/14 42000 Fix test flakes + // 2022/10/27 44000 Update tcmalloc // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -398,7 +399,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSize) { // https://github.com/envoyproxy/envoy/issues/12209 // EXPECT_MEMORY_EQ(m_per_cluster, 37061); } - EXPECT_MEMORY_LE(m_per_cluster, 42000); // Round up to allow platform variations. + EXPECT_MEMORY_LE(m_per_cluster, 44000); // Round up to allow platform variations. } TEST_P(ClusterMemoryTestRunner, MemoryLargeHostSizeWithStats) { From 78a3304d78a23a5de874dd2dd6615e52e91f2686 Mon Sep 17 00:00:00 2001 From: James Fish Date: Tue, 1 Nov 2022 23:24:30 -0400 Subject: [PATCH 073/112] thrift_proxy: Add reloadable feature flag to allow negative field ids (#23670) Add a runtime reloadable feature flag to allow negative field ids: envoy.reloadable_features.thrift_allow_negative_field_ids. Since apache thrift 0.8 the apache thrift compiler has by default disallowed negative field ids, however --allow-neg-keys can be used to support negative field ids introduced in a thrift schemas before 0.8. Unfortunately such old thrift schemas exist, and therefore the thrift_proxy is changed to be generous with what it accepts. Ideally there would be a deprecation window in this but apache thrift hasn't removed the deprecated feature in 11 years. See apache/thrift@c7206a4 Risk Level: low - a closed source filter could depend on positive only field ids or upstream could depend on clients migrating to positive field ids and previously be blocking negative field ids Testing: unit test Docs Changes: N/A Release Notes: Yes Platform Specific Features: No Runtime guard: envoy.reloadable_features.thrift_allow_negative_field_ids Signed-off-by: James Fish --- changelogs/current.yaml | 3 +++ source/common/runtime/runtime_features.cc | 1 + .../filters/network/thrift_proxy/BUILD | 2 ++ .../thrift_proxy/binary_protocol_impl.cc | 21 +++++++++++----- .../thrift_proxy/binary_protocol_impl.h | 2 ++ .../thrift_proxy/compact_protocol_impl.cc | 23 +++++++++++------ .../thrift_proxy/compact_protocol_impl.h | 2 ++ .../filters/network/thrift_proxy/BUILD | 2 ++ .../thrift_proxy/binary_protocol_impl_test.cc | 25 ++++++++++++++++++- .../compact_protocol_impl_test.cc | 25 ++++++++++++++++++- .../network/thrift_proxy/conn_manager_test.cc | 18 +++++++------ 11 files changed, 101 insertions(+), 23 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 55e829e05079..d3086f524856 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -71,6 +71,9 @@ new_features: - area: http change: | allowing the dynamic forward proxy cluster to :ref:`allow_coalesced_connections ` for HTTP/2 and HTTP/3 connections. +- area: thrift_proxy + change: | + added ``envoy.reloadable_features.thrift_allow_negative_field_ids`` to support negative field ids for legacy thrift service. - area: udp_proxy change: | added support for :ref:`proxy_access_log `. diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index b38867e5f29d..4b3cce5d327f 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -62,6 +62,7 @@ RUNTIME_GUARD(envoy_reloadable_features_quic_defer_send_in_response_to_packet); RUNTIME_GUARD(envoy_reloadable_features_skip_delay_close); RUNTIME_GUARD(envoy_reloadable_features_skip_dns_lookup_for_proxied_requests); RUNTIME_GUARD(envoy_reloadable_features_test_feature_true); +RUNTIME_GUARD(envoy_reloadable_features_thrift_allow_negative_field_ids); RUNTIME_GUARD(envoy_reloadable_features_tls_async_cert_validation); RUNTIME_GUARD(envoy_reloadable_features_top_level_ecds_stats); RUNTIME_GUARD(envoy_reloadable_features_udp_proxy_connect); diff --git a/source/extensions/filters/network/thrift_proxy/BUILD b/source/extensions/filters/network/thrift_proxy/BUILD index 3ec89e0d86bd..6e6faafb15d0 100644 --- a/source/extensions/filters/network/thrift_proxy/BUILD +++ b/source/extensions/filters/network/thrift_proxy/BUILD @@ -221,6 +221,7 @@ envoy_cc_library( ":buffer_helper_lib", ":protocol_interface", "//source/common/common:macros", + "//source/common/runtime:runtime_features_lib", ], alwayslink = 1, ) @@ -238,6 +239,7 @@ envoy_cc_library( ":buffer_helper_lib", ":protocol_interface", "//source/common/common:macros", + "//source/common/runtime:runtime_features_lib", ], alwayslink = 1, ) diff --git a/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.cc b/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.cc index 961f89a8277a..625f99cc9da6 100644 --- a/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.cc @@ -7,6 +7,7 @@ #include "source/common/common/assert.h" #include "source/common/common/fmt.h" #include "source/common/common/macros.h" +#include "source/common/runtime/runtime_features.h" #include "source/extensions/filters/network/thrift_proxy/buffer_helper.h" namespace Envoy { @@ -79,14 +80,24 @@ bool BinaryProtocolImpl::peekReplyPayload(Buffer::Instance& buffer, ReplyType& r } int16_t id = buffer.peekBEInt(1); - if (id < 0) { - throw EnvoyException(absl::StrCat("invalid binary protocol field id ", id)); - } + validateFieldId(id); // successful response struct in field id 0, error (IDL exception) in field id greater than 0 reply_type = id == 0 ? ReplyType::Success : ReplyType::Error; return true; } +void BinaryProtocolImpl::validateFieldId(int16_t id) { + if (id >= 0) { + return; + } + + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.thrift_allow_negative_field_ids")) { + return; + } + + throw EnvoyException(absl::StrCat("invalid binary protocol field id ", id)); +} + bool BinaryProtocolImpl::readStructBegin(Buffer::Instance& buffer, std::string& name) { UNREFERENCED_PARAMETER(buffer); name.clear(); // binary protocol does not transmit struct names @@ -115,9 +126,7 @@ bool BinaryProtocolImpl::readFieldBegin(Buffer::Instance& buffer, std::string& n return false; } int16_t id = buffer.peekBEInt(1); - if (id < 0) { - throw EnvoyException(absl::StrCat("invalid binary protocol field id ", id)); - } + validateFieldId(id); field_id = id; buffer.drain(3); } diff --git a/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.h b/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.h index 1550e8d22610..58d1602089bd 100644 --- a/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.h +++ b/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.h @@ -82,6 +82,8 @@ class BinaryProtocolImpl : public Protocol { static constexpr uint64_t MinMessageBeginLength = 12; private: + static void validateFieldId(int16_t id); + const static uint16_t Magic; }; diff --git a/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.cc b/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.cc index 3aef9e205aa7..8515e63dd377 100644 --- a/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.cc @@ -7,6 +7,7 @@ #include "source/common/common/assert.h" #include "source/common/common/fmt.h" #include "source/common/common/macros.h" +#include "source/common/runtime/runtime_features.h" #include "source/extensions/filters/network/thrift_proxy/buffer_helper.h" namespace Envoy { @@ -111,14 +112,25 @@ bool CompactProtocolImpl::peekReplyPayload(Buffer::Instance& buffer, ReplyType& return false; } - if (id < 0 || id > std::numeric_limits::max()) { - throw EnvoyException(absl::StrCat("invalid compact protocol field id ", id)); - } + validateFieldId(id); // successful response struct in field id 0, error (IDL exception) in field id greater than 0 reply_type = id == 0 ? ReplyType::Success : ReplyType::Error; return true; } +void CompactProtocolImpl::validateFieldId(int32_t id) { + if (id >= 0 && id <= std::numeric_limits::max()) { + return; + } + + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.thrift_allow_negative_field_ids") && + id < 0 && id >= std::numeric_limits::min()) { + return; + } + + throw EnvoyException(absl::StrCat("invalid compact protocol field id ", id)); +} + bool CompactProtocolImpl::readStructBegin(Buffer::Instance& buffer, std::string& name) { UNREFERENCED_PARAMETER(buffer); name.clear(); // compact protocol does not transmit struct names @@ -176,10 +188,7 @@ bool CompactProtocolImpl::readFieldBegin(Buffer::Instance& buffer, std::string& return false; } - if (id < 0 || id > std::numeric_limits::max()) { - throw EnvoyException(absl::StrCat("invalid compact protocol field id ", id)); - } - + validateFieldId(id); compact_field_type = static_cast(delta_and_type); compact_field_id = static_cast(id); } else { diff --git a/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.h b/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.h index a2c978bcf2f8..f7435a662243 100644 --- a/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.h +++ b/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.h @@ -98,6 +98,8 @@ class CompactProtocolImpl : public Protocol { void writeFieldBeginInternal(Buffer::Instance& buffer, FieldType field_type, int16_t field_id, absl::optional field_type_override); + static void validateFieldId(int32_t id); + std::stack last_field_id_stack_{}; int16_t last_field_id_{0}; diff --git a/test/extensions/filters/network/thrift_proxy/BUILD b/test/extensions/filters/network/thrift_proxy/BUILD index 438d6ab005da..d5b192c85ee3 100644 --- a/test/extensions/filters/network/thrift_proxy/BUILD +++ b/test/extensions/filters/network/thrift_proxy/BUILD @@ -171,6 +171,7 @@ envoy_extension_cc_test( ":utility_lib", "//source/extensions/filters/network/thrift_proxy:binary_protocol_lib", "//test/test_common:printers_lib", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", ], ) @@ -184,6 +185,7 @@ envoy_extension_cc_test( ":utility_lib", "//source/extensions/filters/network/thrift_proxy:compact_protocol_lib", "//test/test_common:printers_lib", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/extensions/filters/network/thrift_proxy/binary_protocol_impl_test.cc b/test/extensions/filters/network/thrift_proxy/binary_protocol_impl_test.cc index 480a73c1924d..4eda34dcac7e 100644 --- a/test/extensions/filters/network/thrift_proxy/binary_protocol_impl_test.cc +++ b/test/extensions/filters/network/thrift_proxy/binary_protocol_impl_test.cc @@ -5,6 +5,7 @@ #include "test/extensions/filters/network/thrift_proxy/utility.h" #include "test/test_common/printers.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -171,6 +172,7 @@ TEST_F(BinaryProtocolTest, ReadStructEnd) { TEST_F(BinaryProtocolTest, ReadFieldBegin) { BinaryProtocolImpl proto; + TestScopedRuntime scoped_runtime; // Insufficient data { @@ -233,7 +235,7 @@ TEST_F(BinaryProtocolTest, ReadFieldBegin) { EXPECT_EQ(buffer.length(), 0); } - // field id < 0 + // field id < 0 and not allowed negative id { Buffer::OwnedImpl buffer; std::string name = "-"; @@ -242,6 +244,8 @@ TEST_F(BinaryProtocolTest, ReadFieldBegin) { buffer.writeByte(FieldType::I32); buffer.writeBEInt(-1); + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.thrift_allow_negative_field_ids", "false"}}); EXPECT_THROW_WITH_MESSAGE(proto.readFieldBegin(buffer, name, field_type, field_id), EnvoyException, "invalid binary protocol field id -1"); @@ -250,6 +254,25 @@ TEST_F(BinaryProtocolTest, ReadFieldBegin) { EXPECT_EQ(field_id, 1); EXPECT_EQ(buffer.length(), 3); } + + // field id < 0 and allowed negative id + { + Buffer::OwnedImpl buffer; + std::string name = "-"; + FieldType field_type = FieldType::String; + int16_t field_id = 1; + + buffer.writeByte(FieldType::I32); + buffer.writeBEInt(-1); + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.thrift_allow_negative_field_ids", "true"}}); + + EXPECT_TRUE(proto.readFieldBegin(buffer, name, field_type, field_id)); + EXPECT_EQ(name, ""); + EXPECT_EQ(field_type, FieldType::I32); + EXPECT_EQ(field_id, -1); + EXPECT_EQ(buffer.length(), 0); + } } TEST_F(BinaryProtocolTest, ReadFieldEnd) { diff --git a/test/extensions/filters/network/thrift_proxy/compact_protocol_impl_test.cc b/test/extensions/filters/network/thrift_proxy/compact_protocol_impl_test.cc index aef8cb03a9a2..042aaddd2d5e 100644 --- a/test/extensions/filters/network/thrift_proxy/compact_protocol_impl_test.cc +++ b/test/extensions/filters/network/thrift_proxy/compact_protocol_impl_test.cc @@ -5,6 +5,7 @@ #include "test/extensions/filters/network/thrift_proxy/utility.h" #include "test/test_common/printers.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -236,6 +237,7 @@ TEST_F(CompactProtocolTest, ReadStruct) { TEST_F(CompactProtocolTest, ReadFieldBegin) { CompactProtocolImpl proto; + TestScopedRuntime scoped_runtime; // Insufficient data { @@ -325,7 +327,7 @@ TEST_F(CompactProtocolTest, ReadFieldBegin) { EXPECT_EQ(buffer.length(), 4); } - // Long-form field header, field id < 0 + // Long-form field header, field id < 0 and not allowed negative field id { Buffer::OwnedImpl buffer; std::string name = "-"; @@ -335,6 +337,8 @@ TEST_F(CompactProtocolTest, ReadFieldBegin) { buffer.writeByte(0x05); addSeq(buffer, {0x01}); // zigzag(1) = -1 + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.thrift_allow_negative_field_ids", "false"}}); EXPECT_THROW_WITH_MESSAGE(proto.readFieldBegin(buffer, name, field_type, field_id), EnvoyException, "invalid compact protocol field id -1"); EXPECT_EQ(name, "-"); @@ -343,6 +347,25 @@ TEST_F(CompactProtocolTest, ReadFieldBegin) { EXPECT_EQ(buffer.length(), 2); } + // Long-form field header, field id < 0 and allowed negative field id + { + Buffer::OwnedImpl buffer; + std::string name = "-"; + FieldType field_type = FieldType::String; + int16_t field_id = 1; + + buffer.writeByte(0x05); + addSeq(buffer, {0x01}); // zigzag(1) = -1 + + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.thrift_allow_negative_field_ids", "true"}}); + EXPECT_TRUE(proto.readFieldBegin(buffer, name, field_type, field_id)); + EXPECT_EQ(name, ""); + EXPECT_EQ(field_type, FieldType::I32); + EXPECT_EQ(field_id, -1); + EXPECT_EQ(buffer.length(), 0); + } + // Unknown compact protocol field type { Buffer::OwnedImpl buffer; diff --git a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc index 067aa2ccb4e3..8660b335cdd6 100644 --- a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc @@ -815,12 +815,13 @@ TEST_F(ThriftConnectionManagerTest, OnDataHandlesProtocolError) { 0x80, 0x01, 0x00, 0x01, // binary, call 0x00, 0x00, 0x00, 0x04, 'n', 'a', 'm', 'e', // message name 0x00, 0x00, 0x00, 0x01, // sequence id - 0x08, 0xff, 0xff // illegal field id + 0x0d, 0x00, 0x01, 0x0b, 0xb, // map, field id, string key/value + 0xff, 0xff, 0xff, 0xff, // negative length }); - std::string err = "invalid binary protocol field id -1"; + std::string err = "negative binary protocol map size -1"; addSeq(write_buffer_, { - 0x00, 0x00, 0x00, 0x42, // framed: 66 bytes + 0x00, 0x00, 0x00, 0x43, // framed: 67 bytes 0x80, 0x01, 0x00, 0x03, // binary, exception 0x00, 0x00, 0x00, 0x04, 'n', 'a', 'm', 'e', // message name 0x00, 0x00, 0x00, 0x01, // sequence id @@ -1262,13 +1263,14 @@ TEST_F(ThriftConnectionManagerTest, RequestAndResponseProtocolError) { EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request_call").value()); - // illegal field id + // illegal negative set length addSeq(write_buffer_, { - 0x00, 0x00, 0x00, 0x1f, // framed: 31 bytes + 0x00, 0x00, 0x00, 0x2f, // framed: 31 bytes 0x80, 0x01, 0x00, 0x02, // binary, reply 0x00, 0x00, 0x00, 0x04, 'n', 'a', 'm', 'e', // message name 0x00, 0x00, 0x00, 0x01, // sequence id - 0x08, 0xff, 0xff // illegal field id + 0x0e, 0x00, 0x00, 0x0b, // set, field id, strings + 0xff, 0xff, 0xff, 0xff, // negative length }); FramedTransportImpl transport; @@ -1292,8 +1294,8 @@ TEST_F(ThriftConnectionManagerTest, RequestAndResponseProtocolError) { EXPECT_EQ(0U, store_.counter("test.response_error").value()); EXPECT_EQ(1U, store_.counter("test.response_decoding_error").value()); - EXPECT_EQ(access_log_data_, - "name cluster passthrough_enabled=false framed binary call - - - - 0 0 0 -\n"); + EXPECT_EQ(access_log_data_, "name cluster passthrough_enabled=false framed binary call framed " + "binary reply success 0 0 0 -\n"); } TEST_F(ThriftConnectionManagerTest, RequestAndTransportApplicationException) { From 2799c91ca5b3e3e8340ad55c39300b88af64415e Mon Sep 17 00:00:00 2001 From: code Date: Wed, 2 Nov 2022 14:38:18 +0800 Subject: [PATCH 074/112] generic proxy: make sure the execute order of encoder filters is the reverse of decoder filters' (#23770) * generic proxy: make sure the execuate order of encoder filters is the reverse of decoderfilters' Signed-off-by: wbpcode --- changelogs/current.yaml | 3 ++ .../filters/network/source/proxy.cc | 11 +++++-- .../filters/network/source/proxy.h | 2 ++ .../filters/network/test/proxy_test.cc | 30 +++++++++++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index d3086f524856..801105a42778 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -23,6 +23,9 @@ bug_fixes: - area: oauth2 change: | fixed a bug when passthrough header was matched, envoy would always remove the authorization header. This behavioral change can be temporarily reverted by setting runtime guard ``envoy.reloadable_features.oauth_header_passthrough_fix`` to false. +- area: generic_proxy + change: | + fixed a bug that encoder filters and decoder filters of generic proxy will be executed in the same order. The encoder filters' execuate order should be the reverse of decoder filters' in the generic proxy. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/contrib/generic_proxy/filters/network/source/proxy.cc b/contrib/generic_proxy/filters/network/source/proxy.cc index 6bfb9fe4649c..a2cf09d5b5b1 100644 --- a/contrib/generic_proxy/filters/network/source/proxy.cc +++ b/contrib/generic_proxy/filters/network/source/proxy.cc @@ -164,6 +164,13 @@ void ActiveStream::onEncodingSuccess(Buffer::Instance& buffer, bool close_connec parent_.connection().write(buffer, close_connection); } +void ActiveStream::initializeFilterChain(FilterChainFactory& factory) { + factory.createFilterChain(*this); + // Reverse the encoder filter chain so that the first encoder filter is the last filter in the + // chain. + std::reverse(encoder_filters_.begin(), encoder_filters_.end()); +} + Envoy::Network::FilterStatus Filter::onData(Envoy::Buffer::Instance& data, bool) { if (downstream_connection_closed_) { return Envoy::Network::FilterStatus::StopIteration; @@ -189,8 +196,8 @@ void Filter::newDownstreamRequest(RequestPtr request) { auto raw_stream = stream.get(); LinkedList::moveIntoList(std::move(stream), active_streams_); - config_->createFilterChain(*raw_stream); - + // Initialize filter chian. + raw_stream->initializeFilterChain(*config_); // Start request. raw_stream->continueDecoding(); } diff --git a/contrib/generic_proxy/filters/network/source/proxy.h b/contrib/generic_proxy/filters/network/source/proxy.h index 72a48a527885..f054584b08a6 100644 --- a/contrib/generic_proxy/filters/network/source/proxy.h +++ b/contrib/generic_proxy/filters/network/source/proxy.h @@ -193,6 +193,8 @@ class ActiveStream : public FilterChainManager, encoder_filters_.emplace_back(std::move(filter)); } + void initializeFilterChain(FilterChainFactory& factory); + Envoy::Event::Dispatcher& dispatcher(); const CodecFactory& downstreamCodec(); void resetStream(); diff --git a/contrib/generic_proxy/filters/network/test/proxy_test.cc b/contrib/generic_proxy/filters/network/test/proxy_test.cc index 92cc5154c780..aedf700c8c89 100644 --- a/contrib/generic_proxy/filters/network/test/proxy_test.cc +++ b/contrib/generic_proxy/filters/network/test/proxy_test.cc @@ -580,6 +580,36 @@ TEST_F(FilterTest, ActiveStreamAddFilters) { EXPECT_EQ(0, active_stream->nextEncoderFilterIndexForTest()); } +TEST_F(FilterTest, ActiveStreamAddFiltersOrder) { + auto filter_0 = std::make_shared>(); + auto filter_1 = std::make_shared>(); + auto filter_2 = std::make_shared>(); + + mock_stream_filters_ = {{"fake_test_filter_name_0", filter_0}, + {"fake_test_filter_name_1", filter_1}, + {"fake_test_filter_name_2", filter_2}}; + + initializeFilter(); + + auto request = std::make_unique(); + + filter_->newDownstreamRequest(std::move(request)); + EXPECT_EQ(1, filter_->activeStreamsForTest().size()); + + auto active_stream = filter_->activeStreamsForTest().begin()->get(); + + EXPECT_EQ(3, active_stream->decoderFiltersForTest().size()); + EXPECT_EQ(3, active_stream->encoderFiltersForTest().size()); + + EXPECT_EQ(filter_0.get(), active_stream->decoderFiltersForTest()[0]->filter_.get()); + EXPECT_EQ(filter_1.get(), active_stream->decoderFiltersForTest()[1]->filter_.get()); + EXPECT_EQ(filter_2.get(), active_stream->decoderFiltersForTest()[2]->filter_.get()); + + EXPECT_EQ(filter_2.get(), active_stream->encoderFiltersForTest()[0]->filter_.get()); + EXPECT_EQ(filter_1.get(), active_stream->encoderFiltersForTest()[1]->filter_.get()); + EXPECT_EQ(filter_0.get(), active_stream->encoderFiltersForTest()[2]->filter_.get()); +} + TEST_F(FilterTest, ActiveStreamFiltersContinueDecoding) { auto mock_stream_filter_0 = std::make_shared>(); auto mock_stream_filter_1 = std::make_shared>(); From 883f63507aa214dd22346aa0cc8c93ebadc74fdc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 09:07:39 +0000 Subject: [PATCH 075/112] build(deps): bump mysql from `06314a7` to `d405545` in /examples/mysql (#23738) Bumps mysql from `06314a7` to `d405545`. --- updated-dependencies: - dependency-name: mysql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/mysql/Dockerfile-mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mysql/Dockerfile-mysql b/examples/mysql/Dockerfile-mysql index d59d5c1bbcb8..281ba87b0398 100644 --- a/examples/mysql/Dockerfile-mysql +++ b/examples/mysql/Dockerfile-mysql @@ -1 +1 @@ -FROM mysql:8.0.31@sha256:06314a7a220f6043436cfd72fd9c7f174fd58ef69fe4b788625fa53be4ab66aa +FROM mysql:8.0.31@sha256:d4055451e7f42869e64089a60d1abc9e66eccde2910629f0dd666b53a5f230d8 From e5120b4bd294262989ac942c0d0ec3d3001cf71f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 09:08:04 +0000 Subject: [PATCH 076/112] build(deps): bump pytz from 2022.5 to 2022.6 in /tools/dependency (#23776) Bumps [pytz](https://github.com/stub42/pytz) from 2022.5 to 2022.6. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2022.5...release_2022.6) --- updated-dependencies: - dependency-name: pytz dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/dependency/requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/dependency/requirements.txt b/tools/dependency/requirements.txt index 6eab8f9ab073..3a6ec1e22830 100644 --- a/tools/dependency/requirements.txt +++ b/tools/dependency/requirements.txt @@ -210,8 +210,9 @@ pyparsing==3.0.9 \ # via # -r requirements.in # packaging -pytz==2022.5 \ - --hash=sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22 +pytz==2022.6 \ + --hash=sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427 \ + --hash=sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2 # via -r requirements.in requests==2.28.1 \ --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ From d94c2a788d7880070dc373c97a3c7a090b038fac Mon Sep 17 00:00:00 2001 From: zackzhangkai Date: Wed, 2 Nov 2022 17:08:38 +0800 Subject: [PATCH 077/112] fix comments (#23788) Signed-off-by: zackzhangkai --- examples/grpc-bridge/docker-compose-protos.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/grpc-bridge/docker-compose-protos.yaml b/examples/grpc-bridge/docker-compose-protos.yaml index beaf8567c011..3199dc29cadd 100644 --- a/examples/grpc-bridge/docker-compose-protos.yaml +++ b/examples/grpc-bridge/docker-compose-protos.yaml @@ -1,7 +1,5 @@ version: "3.8" -# This is the conversion from a script to a dockerized version of the script -# https://github.com/envoyproxy/envoy/blob/main/examples/grpc-bridge/service/script/gen services: # $ docker run -ti -v $(pwd):/protos -v $(pwd)/stubs:/stubs grpc/go protoc --go_out=plugins=grpc:/stubs -I/protos /protos/kv.proto From f50fe9c1fd20f23c4fe6f202c541d6cfa231ac5c Mon Sep 17 00:00:00 2001 From: Kuo-Chung Hsu Date: Wed, 2 Nov 2022 05:41:27 -0700 Subject: [PATCH 078/112] test: clean up unused variable (#23679) Signed-off-by: kuochunghsu --- test/common/tcp/conn_pool_test.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/test/common/tcp/conn_pool_test.cc b/test/common/tcp/conn_pool_test.cc index 8adfa7ad03c9..f6870ecabc36 100644 --- a/test/common/tcp/conn_pool_test.cc +++ b/test/common/tcp/conn_pool_test.cc @@ -1058,7 +1058,6 @@ TEST_F(TcpConnPoolImplTest, TestIdleTimeout) { c1.releaseConn(); conn_pool_->test_conns_[0].connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); - testing::MockFunction drained_callback; EXPECT_CALL(idle_callback, Call()); conn_pool_->drainConnections(Envoy::ConnectionPool::DrainBehavior::DrainAndDelete); EXPECT_CALL(*conn_pool_, onConnDestroyedForTest()); From 1d60a116413a0422b2df50e5f6ef8b553caba53b Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 2 Nov 2022 09:06:03 -0400 Subject: [PATCH 079/112] moving original_dst cluster to extensions (#23694) Any Envoy users who customize their pre-built extensions will need to evaluate if they need this cluster. Risk Level: medium Testing: n/a Docs Changes: n/a Release Notes: inline Signed-off-by: Alyssa Wilk --- CODEOWNERS | 2 ++ api/envoy/config/cluster/v3/cluster.proto | 1 + changelogs/current.yaml | 3 ++ source/common/upstream/BUILD | 22 ------------ .../common/upstream/cluster_manager_impl.cc | 2 +- source/common/upstream/upstream_impl.cc | 2 +- source/extensions/clusters/original_dst/BUILD | 31 ++++++++++++++++ .../original_dst}/original_dst_cluster.cc | 2 +- .../original_dst}/original_dst_cluster.h | 0 source/extensions/extensions_build_config.bzl | 1 + source/extensions/extensions_metadata.yaml | 5 +++ test/common/upstream/BUILD | 27 +------------- test/extensions/clusters/original_dst/BUILD | 35 +++++++++++++++++++ .../original_dst_cluster_test.cc | 2 +- .../filters/listener/original_dst/BUILD | 1 + .../filters/listener/proxy_protocol/BUILD | 1 + test/integration/BUILD | 1 + test/server/config_validation/BUILD | 1 + tools/code_format/config.yaml | 1 + 19 files changed, 88 insertions(+), 52 deletions(-) create mode 100644 source/extensions/clusters/original_dst/BUILD rename source/{common/upstream => extensions/clusters/original_dst}/original_dst_cluster.cc (99%) rename source/{common/upstream => extensions/clusters/original_dst}/original_dst_cluster.h (100%) create mode 100644 test/extensions/clusters/original_dst/BUILD rename test/{common/upstream => extensions/clusters/original_dst}/original_dst_cluster_test.cc (99%) diff --git a/CODEOWNERS b/CODEOWNERS index b30e99bf10e2..36637fadd416 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -278,6 +278,8 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/filters/common @UNOWNED @UNOWNED /*/extensions/filters/http/common @UNOWNED @UNOWNED /*/extensions/filters/network/common @UNOWNED @UNOWNED +/*/extensions/clusters/original_dst @UNOWNED @UNOWNED + # URL Pattern Match and Rewrite Library /*/extensions/path/uri_template_lib @alyssawilk @yanjunxiang-google diff --git a/api/envoy/config/cluster/v3/cluster.proto b/api/envoy/config/cluster/v3/cluster.proto index 79c9909b4078..e9438bd5f6d1 100644 --- a/api/envoy/config/cluster/v3/cluster.proto +++ b/api/envoy/config/cluster/v3/cluster.proto @@ -525,6 +525,7 @@ message Cluster { // Specific configuration for the // :ref:`Original Destination ` // load balancing policy. + // [#extension: envoy.clusters.original_dst] message OriginalDstLbConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Cluster.OriginalDstLbConfig"; diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 801105a42778..af22abbd41c7 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -2,6 +2,9 @@ date: Pending behavior_changes: # *Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* +- area: build + change: | + moved the original_dst cluster to extensions. If you use this cluster and override extensions_build_config.bzl you will now need to include it explicitly. minor_behavior_changes: # *Changes that may cause incompatibilities for some users, but should not for most* diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 117589ffd01d..947bb777f66c 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -364,27 +364,6 @@ envoy_cc_library( alwayslink = LEGACY_ALWAYSLINK, ) -envoy_cc_library( - name = "original_dst_cluster_lib", - srcs = ["original_dst_cluster.cc"], - hdrs = ["original_dst_cluster.h"], - deps = [ - ":cluster_factory_lib", - ":upstream_includes", - "//envoy/secret:secret_manager_interface", - "//envoy/upstream:cluster_factory_interface", - "//source/common/common:empty_string", - "//source/common/network:address_lib", - "//source/common/network:filter_state_dst_address_lib", - "//source/common/network:utility_lib", - "//source/common/runtime:runtime_features_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", - ], - alwayslink = LEGACY_ALWAYSLINK, -) - envoy_cc_library( name = "outlier_detection_lib", srcs = ["outlier_detection_impl.cc"], @@ -541,7 +520,6 @@ envoy_cc_library( ":health_checker_lib", # TODO(mattklein123): Move the clusters to extensions so they can be compiled out. ":logical_dns_cluster_lib", - ":original_dst_cluster_lib", ":static_cluster_lib", ":strict_dns_cluster_lib", ":upstream_includes", diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index c676fac636f3..d89d9fe10ec6 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -39,9 +39,9 @@ #include "source/common/runtime/runtime_features.h" #include "source/common/tcp/conn_pool.h" #include "source/common/upstream/cds_api_impl.h" +#include "source/common/upstream/cluster_factory_impl.h" #include "source/common/upstream/load_balancer_impl.h" #include "source/common/upstream/maglev_lb.h" -#include "source/common/upstream/original_dst_cluster.h" #include "source/common/upstream/priority_conn_pool_map_impl.h" #include "source/common/upstream/ring_hash_lb.h" #include "source/common/upstream/subset_lb.h" diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index b6bc7cc4f8e3..e27ea8e35b64 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -48,10 +48,10 @@ #include "source/common/router/config_utility.h" #include "source/common/runtime/runtime_features.h" #include "source/common/runtime/runtime_impl.h" +#include "source/common/upstream/cluster_factory_impl.h" #include "source/common/upstream/eds.h" #include "source/common/upstream/health_checker_impl.h" #include "source/common/upstream/logical_dns_cluster.h" -#include "source/common/upstream/original_dst_cluster.h" #include "source/extensions/filters/network/http_connection_manager/config.h" #include "source/server/transport_socket_config_impl.h" diff --git a/source/extensions/clusters/original_dst/BUILD b/source/extensions/clusters/original_dst/BUILD new file mode 100644 index 000000000000..72a2ca934460 --- /dev/null +++ b/source/extensions/clusters/original_dst/BUILD @@ -0,0 +1,31 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "original_dst_cluster_lib", + srcs = ["original_dst_cluster.cc"], + hdrs = ["original_dst_cluster.h"], + # prevously considered core code. + visibility = ["//visibility:public"], + deps = [ + "//envoy/secret:secret_manager_interface", + "//envoy/upstream:cluster_factory_interface", + "//source/common/common:empty_string", + "//source/common/network:address_lib", + "//source/common/network:filter_state_dst_address_lib", + "//source/common/network:utility_lib", + "//source/common/runtime:runtime_features_lib", + "//source/common/upstream:cluster_factory_lib", + "//source/common/upstream:upstream_includes", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + ], +) diff --git a/source/common/upstream/original_dst_cluster.cc b/source/extensions/clusters/original_dst/original_dst_cluster.cc similarity index 99% rename from source/common/upstream/original_dst_cluster.cc rename to source/extensions/clusters/original_dst/original_dst_cluster.cc index a5509585ea44..32dc5997747c 100644 --- a/source/common/upstream/original_dst_cluster.cc +++ b/source/extensions/clusters/original_dst/original_dst_cluster.cc @@ -1,4 +1,4 @@ -#include "source/common/upstream/original_dst_cluster.h" +#include "source/extensions/clusters/original_dst/original_dst_cluster.h" #include #include diff --git a/source/common/upstream/original_dst_cluster.h b/source/extensions/clusters/original_dst/original_dst_cluster.h similarity index 100% rename from source/common/upstream/original_dst_cluster.h rename to source/extensions/clusters/original_dst/original_dst_cluster.h diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 2169d2eb42f7..544329f8e327 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -20,6 +20,7 @@ EXTENSIONS = { "envoy.clusters.aggregate": "//source/extensions/clusters/aggregate:cluster", "envoy.clusters.dynamic_forward_proxy": "//source/extensions/clusters/dynamic_forward_proxy:cluster", "envoy.clusters.redis": "//source/extensions/clusters/redis:redis_cluster", + "envoy.clusters.original_dst": "//source/extensions/clusters/original_dst:original_dst_cluster_lib", # # Compression diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index e0fcb8dd201d..e6a548b2f629 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -85,6 +85,11 @@ envoy.clusters.dynamic_forward_proxy: - envoy.clusters security_posture: robust_to_untrusted_downstream status: stable +envoy.clusters.original_dst: + categories: + - envoy.clusters + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable envoy.clusters.redis: categories: - envoy.clusters diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index cb40b965b45b..a7ff4bf10d1b 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -72,6 +72,7 @@ envoy_cc_test( ":test_cluster_manager", "//source/common/router:context_lib", "//source/common/upstream:load_balancer_factory_base_lib", + "//source/extensions/clusters/original_dst:original_dst_cluster_lib", "//source/extensions/network/dns_resolver/cares:config", "//source/extensions/transport_sockets/tls:config", "//test/config:v2_link_hacks", @@ -469,32 +470,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "original_dst_cluster_test", - srcs = ["original_dst_cluster_test.cc"], - deps = [ - ":utility_lib", - "//source/common/event:dispatcher_lib", - "//source/common/network:filter_state_dst_address_lib", - "//source/common/network:utility_lib", - "//source/common/upstream:original_dst_cluster_lib", - "//source/common/upstream:upstream_lib", - "//source/extensions/transport_sockets/raw_buffer:config", - "//test/mocks:common_lib", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/network:network_mocks", - "//test/mocks/protobuf:protobuf_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/mocks/server:admin_mocks", - "//test/mocks/server:instance_mocks", - "//test/mocks/ssl:ssl_mocks", - "//test/mocks/upstream:cluster_manager_mocks", - "//test/test_common:test_runtime_lib", - "//test/test_common:utility_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "outlier_detection_impl_test", srcs = ["outlier_detection_impl_test.cc"], diff --git a/test/extensions/clusters/original_dst/BUILD b/test/extensions/clusters/original_dst/BUILD new file mode 100644 index 000000000000..9f8845785b71 --- /dev/null +++ b/test/extensions/clusters/original_dst/BUILD @@ -0,0 +1,35 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "original_dst_cluster_test", + srcs = ["original_dst_cluster_test.cc"], + deps = [ + "//source/common/event:dispatcher_lib", + "//source/common/network:filter_state_dst_address_lib", + "//source/common/network:utility_lib", + "//source/common/upstream:upstream_lib", + "//source/extensions/clusters/original_dst:original_dst_cluster_lib", + "//source/extensions/transport_sockets/raw_buffer:config", + "//test/common/upstream:utility_lib", + "//test/mocks:common_lib", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/network:network_mocks", + "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/mocks/server:admin_mocks", + "//test/mocks/server:instance_mocks", + "//test/mocks/ssl:ssl_mocks", + "//test/mocks/upstream:cluster_manager_mocks", + "//test/test_common:test_runtime_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + ], +) diff --git a/test/common/upstream/original_dst_cluster_test.cc b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc similarity index 99% rename from test/common/upstream/original_dst_cluster_test.cc rename to test/extensions/clusters/original_dst/original_dst_cluster_test.cc index c8ab1771f720..6e465ab77d65 100644 --- a/test/common/upstream/original_dst_cluster_test.cc +++ b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc @@ -12,8 +12,8 @@ #include "source/common/network/filter_state_dst_address.h" #include "source/common/network/utility.h" #include "source/common/singleton/manager_impl.h" -#include "source/common/upstream/original_dst_cluster.h" #include "source/common/upstream/upstream_impl.h" +#include "source/extensions/clusters/original_dst/original_dst_cluster.h" #include "source/server/transport_socket_config_impl.h" #include "test/common/upstream/utility.h" diff --git a/test/extensions/filters/listener/original_dst/BUILD b/test/extensions/filters/listener/original_dst/BUILD index 392e77d4de57..311946e1ad2f 100644 --- a/test/extensions/filters/listener/original_dst/BUILD +++ b/test/extensions/filters/listener/original_dst/BUILD @@ -26,6 +26,7 @@ envoy_cc_test( ], deps = [ "//source/common/http:header_map_lib", + "//source/extensions/clusters/original_dst:original_dst_cluster_lib", "//source/extensions/filters/http/buffer:config", "//source/extensions/filters/listener/original_dst:config", "//source/extensions/filters/network/tcp_proxy:config", diff --git a/test/extensions/filters/listener/proxy_protocol/BUILD b/test/extensions/filters/listener/proxy_protocol/BUILD index fc156709f7b3..2fb71747be6d 100644 --- a/test/extensions/filters/listener/proxy_protocol/BUILD +++ b/test/extensions/filters/listener/proxy_protocol/BUILD @@ -71,6 +71,7 @@ envoy_extension_cc_test( "//source/common/buffer:buffer_lib", "//source/common/http:codec_client_lib", "//source/extensions/access_loggers/file:config", + "//source/extensions/clusters/original_dst:original_dst_cluster_lib", "//source/extensions/filters/listener/proxy_protocol:config", "//source/extensions/filters/network/tcp_proxy:config", "//test/integration:http_integration_lib", diff --git a/test/integration/BUILD b/test/integration/BUILD index 345ef2dae15c..fac2cb27fafd 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1063,6 +1063,7 @@ envoy_cc_test( "//envoy/registry", "//source/common/http:header_map_lib", "//source/common/http:headers_lib", + "//source/extensions/clusters/original_dst:original_dst_cluster_lib", "//test/integration/filters:clear_route_cache_filter_lib", "//test/integration/filters:encoder_decoder_buffer_filter_lib", "//test/integration/filters:invalid_header_filter_lib", diff --git a/test/server/config_validation/BUILD b/test/server/config_validation/BUILD index d4fa2f33beba..667b656c2e9f 100644 --- a/test/server/config_validation/BUILD +++ b/test/server/config_validation/BUILD @@ -51,6 +51,7 @@ envoy_cc_test( ], env = {"EXAMPLE_CONFIGS_TAR_PATH": "envoy/configs/example_configs.tar"}, deps = [ + "//source/extensions/clusters/original_dst:original_dst_cluster_lib", "//source/extensions/filters/http/router:config", "//source/extensions/filters/listener/original_dst:config", "//source/extensions/filters/network/http_connection_manager:config", diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index 04e1647c4869..b3eacbb5186d 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -290,6 +290,7 @@ unsorted_flags: # https://github.com/envoyproxy/envoy/issues/9953 # PLEASE DO NOT ADD FILES TO THIS LIST WITHOUT SENIOR MAINTAINER APPROVAL visibility_excludes: +- source/extensions/clusters/original_dst/ - source/extensions/early_data/BUILD - source/extensions/filters/http/buffer/BUILD - source/extensions/filters/network/common/BUILD From b56aa061440937c1cf25aa43b16c700743512ed7 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 2 Nov 2022 14:51:29 +0000 Subject: [PATCH 080/112] deps: Bump `com_github_google_flatbuffers` -> 22.10.26 (#23793) Fix #23705 Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index ae438af5749e..63573c9ae9e7 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1133,8 +1133,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "FlatBuffers", project_desc = "Cross platform serialization library architected for maximum memory efficiency", project_url = "https://github.com/google/flatbuffers", - version = "22.9.29", - sha256 = "372df01795c670f6538055a7932fc7eb3e81b3653be4a216c081e9c3c26b1b6d", + version = "22.10.26", + sha256 = "34f1820cfd78a3d92abc880fbb1a644c7fb31a71238995f4ed6b5915a1ad4e79", strip_prefix = "flatbuffers-{version}", urls = ["https://github.com/google/flatbuffers/archive/v{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -1150,7 +1150,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.stat_sinks.wasm", "envoy.rbac.matchers.upstream_ip_port", ], - release_date = "2022-09-30", + release_date = "2022-10-26", cpe = "cpe:2.3:a:google:flatbuffers:*", license = "Apache-2.0", license_url = "https://github.com/google/flatbuffers/blob/v{version}/LICENSE.txt", From 9db77e66e208f5bc72a3155cbf3f9dbd87cd6e12 Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Wed, 2 Nov 2022 11:04:07 -0400 Subject: [PATCH 081/112] fixing OSS filter_fuzz_test OOM issue when thread_count is 800 million. (#23782) Signed-off-by: Yanjun Xiang --- .../extensions/common/async_files/v3/async_file_manager.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/extensions/common/async_files/v3/async_file_manager.proto b/api/envoy/extensions/common/async_files/v3/async_file_manager.proto index 89d840850553..bbe531f6e409 100644 --- a/api/envoy/extensions/common/async_files/v3/async_file_manager.proto +++ b/api/envoy/extensions/common/async_files/v3/async_file_manager.proto @@ -22,7 +22,7 @@ message AsyncFileManagerConfig { // The number of threads to use. If unset or zero, will default to the number // of concurrent threads the hardware supports. This default is subject to // change if performance analysis suggests it. - uint32 thread_count = 1; + uint32 thread_count = 1 [(validate.rules).uint32 = {lte: 1024}]; } // An optional identifier for the manager. An empty string is a valid identifier From 158a28b87cb3dda336392e5ca51941d532cfb8dd Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 2 Nov 2022 15:32:27 +0000 Subject: [PATCH 082/112] deps: Bump `com_github_google_perfetto` -> 30.0 (#23795) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 63573c9ae9e7..e7689a501413 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -192,12 +192,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Perfetto", project_desc = "Perfetto Tracing SDK", project_url = "https://perfetto.dev/", - version = "28.0", - sha256 = "06eec38d02f99d225cdad9444102e77d9da717f8cc55f84a3b212abe94a5fc5a", + version = "30.0", + sha256 = "d1883793a2adb2a4105fc083478bf781badd566d72da45caa99095b61f938a2e", strip_prefix = "perfetto-{version}/sdk", urls = ["https://github.com/google/perfetto/archive/v{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2022-08-02", + release_date = "2022-10-06", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/google/perfetto/blob/v{version}/LICENSE", From f5b42d75d223bc6da3755a1928d266e931c7c80c Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 2 Nov 2022 12:42:44 -0400 Subject: [PATCH 083/112] mobile: starting the repo merge with the assertion filter (#23761) This PR sets up the directory structure, visibility rules, codeowners, adds E-M C++ tests to Envoy CI, and moves one extension over as a proof of concept. Risk Level: low Testing: ported test to Envoy CI. did not turn up platform CI yet Release Notes: inline Part of #23758 Signed-off-by: Alyssa Wilk --- BUILD | 7 + CODEOWNERS | 3 + OWNERS.md | 19 +- bazel/envoy_build_system.bzl | 3 + changelogs/current.yaml | 3 + ci/do_ci.sh | 2 +- ci/run_clang_tidy.sh | 7 +- .../extensions/filters/http/assertion/BUILD | 44 ++ .../filters/http/assertion/config.cc | 29 + .../filters/http/assertion/config.h | 34 ++ .../filters/http/assertion/filter.cc | 224 +++++++ .../filters/http/assertion/filter.h | 56 ++ .../filters/http/assertion/filter.proto | 16 + .../extensions/filters/http/assertion/BUILD | 21 + .../http/assertion/assertion_filter_test.cc | 562 ++++++++++++++++++ source/extensions/extensions_build_config.bzl | 4 +- tools/code_format/envoy_build_fixer.py | 5 + 17 files changed, 1029 insertions(+), 10 deletions(-) create mode 100644 mobile/library/common/extensions/filters/http/assertion/BUILD create mode 100644 mobile/library/common/extensions/filters/http/assertion/config.cc create mode 100644 mobile/library/common/extensions/filters/http/assertion/config.h create mode 100644 mobile/library/common/extensions/filters/http/assertion/filter.cc create mode 100644 mobile/library/common/extensions/filters/http/assertion/filter.h create mode 100644 mobile/library/common/extensions/filters/http/assertion/filter.proto create mode 100644 mobile/test/common/extensions/filters/http/assertion/BUILD create mode 100644 mobile/test/common/extensions/filters/http/assertion/assertion_filter_test.cc diff --git a/BUILD b/BUILD index b4c9a2907ad8..8e5e07c3073c 100644 --- a/BUILD +++ b/BUILD @@ -58,3 +58,10 @@ package_group( "//examples/...", ], ) + +package_group( + name = "mobile_library", + packages = [ + "//mobile/...", + ], +) diff --git a/CODEOWNERS b/CODEOWNERS index 36637fadd416..c5e135791b8d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -285,6 +285,9 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/path/uri_template_lib @alyssawilk @yanjunxiang-google /*/extensions/path/uri_template_lib/proto @alyssawilk @yanjunxiang-google +# mobile +/mobile/ @jpsim @Augustyniak @RyanTheOptimist @alyssawilk @abeyad + # Contrib /contrib/exe/ @mattklein123 @lizan /contrib/client_ssl_auth/ @UNOWNED @UNOWNED diff --git a/OWNERS.md b/OWNERS.md index 0f5bb3550428..1609d87e7592 100644 --- a/OWNERS.md +++ b/OWNERS.md @@ -13,7 +13,7 @@ routing PRs, questions, etc. to the right place. * xDS APIs, configuration and control plane. * Alyssa Wilk ([alyssawilk](https://github.com/alyssawilk)) (alyssar@google.com) * HTTP, flow control, cluster manager, load balancing, and core networking (listeners, - connections, etc.). + connections, etc.), Envoy Mobile. * Stephan Zuercher ([zuercher](https://github.com/zuercher)) (zuercher@gmail.com) * Load balancing, upstream clusters and cluster manager, logging, complex HTTP routing (metadata, etc.), and macOS build. @@ -29,7 +29,7 @@ routing PRs, questions, etc. to the right place. * Ryan Northey ([phlax](https://github.com/phlax)) (ryan@synca.io) * Docs, tooling, CI, containers and sandbox examples * Ryan Hamilton ([RyanTheOptimist](https://github.com/ryantheoptimist)) (rch@google.com) - * HTTP/3, upstream connection management. + * HTTP/3, upstream connection management, Envoy Mobile. # Maintainers @@ -50,6 +50,17 @@ routing PRs, questions, etc. to the right place. * Kuat Yessenov ([kyessenov](https://github.com/kyessenov)) (kuat@google.com) * Listeners, RBAC, CEL, matching, Istio. +# Envoy mobile maintainers + +The following Envoy maintainers have final say over any changes only affecting /mobile + +* JP Simard ([jpsim](https://github.com/jpsim)) (jp@lyft.com) + * iOS (swift/objective-c) platform bindings. +* Rafal Augustyniak ([Augustyniak](https://github.com/Augustyniak)) (raugustyniak@lyft.com) + * iOS (swift/objective-c) platform bindings. +* Ali Beyad ([abeyad](https://github.com/abeyad)) (abeyad@google.com) + * xDS, C++ integration tests. + # Senior extension maintainers The following extension maintainers have final say over the extensions mentioned below. Once they @@ -60,10 +71,6 @@ without further review. * Wasm * Raúl Gutiérrez Segalés ([rgs1](https://github.com/rgs1)) (rgs@pinterest.com) * Thrift -* Ryan Hamilton ([RyanTheOptimist](https://github.com/ryantheoptimist)) (rch@google.com) - * HTTP/3 -* Baiping Wang ([wbpcode](https://github.com/wbpcode)) (wbphub@live.com) - * Dubbo # Envoy security team diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index f87903151df8..8bd4401f7402 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -67,6 +67,9 @@ def envoy_extension_package(enabled_default = True, default_visibility = EXTENSI flag_values = {":enabled": "True"}, ) +def envoy_mobile_package(): + envoy_extension_package() + def envoy_contrib_package(): envoy_extension_package(default_visibility = CONTRIB_EXTENSION_PACKAGE_VISIBILITY) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index af22abbd41c7..b253d5ecae2b 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -83,5 +83,8 @@ new_features: - area: udp_proxy change: | added support for :ref:`proxy_access_log `. +- area: mobile + change: | + started merging the Envoy mobile library into the main Envoy repo. deprecated: diff --git a/ci/do_ci.sh b/ci/do_ci.sh index b179df0c8e0a..034b35a6e644 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -208,7 +208,7 @@ else elif [[ "${CI_TARGET}" == "bazel.msan" ]]; then COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "-//test/extensions/...") fi - TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "@com_github_google_quiche//:ci_tests") + TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "@com_github_google_quiche//:ci_tests" "//mobile/test/...") fi if [[ "$CI_TARGET" == "bazel.release" ]]; then diff --git a/ci/run_clang_tidy.sh b/ci/run_clang_tidy.sh index 44868a62f57c..fd08f1dc78a5 100755 --- a/ci/run_clang_tidy.sh +++ b/ci/run_clang_tidy.sh @@ -76,8 +76,13 @@ function exclude_wasm_examples() { grep -v examples/wasm } +# Exclude envoy mobile. +function exclude_envoy_mobile() { + grep -v mobile/library | grep -v mobile/test +} + function filter_excludes() { - exclude_check_format_testdata | exclude_win32_impl | exclude_macos_impl | exclude_third_party | exclude_wasm_emscripten | exclude_wasm_sdk | exclude_wasm_host | exclude_wasm_test_data | exclude_wasm_examples + exclude_envoy_mobile | exclude_check_format_testdata | exclude_win32_impl | exclude_macos_impl | exclude_third_party | exclude_wasm_emscripten | exclude_wasm_sdk | exclude_wasm_host | exclude_wasm_test_data | exclude_wasm_examples } function run_clang_tidy() { diff --git a/mobile/library/common/extensions/filters/http/assertion/BUILD b/mobile/library/common/extensions/filters/http/assertion/BUILD new file mode 100644 index 000000000000..39d6d5077b08 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/assertion/BUILD @@ -0,0 +1,44 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_mobile_package", + "envoy_proto_library", +) + +licenses(["notice"]) # Apache 2 + +envoy_mobile_package() + +envoy_proto_library( + name = "filter", + srcs = ["filter.proto"], + deps = [ + "@envoy_api//envoy/config/common/matcher/v3:pkg", + ], +) + +envoy_cc_library( + name = "assertion_filter_lib", + srcs = ["filter.cc"], + hdrs = ["filter.h"], + repository = "@envoy", + deps = [ + "filter_cc_proto", + "//envoy/http:codes_interface", + "//envoy/http:filter_interface", + "//source/common/http:header_map_lib", + "//source/extensions/common/matcher:matcher_lib", + "//source/extensions/filters/http/common:pass_through_filter_lib", + ], +) + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + repository = "@envoy", + deps = [ + ":assertion_filter_lib", + "//source/extensions/filters/http/common:factory_base_lib", + ], +) diff --git a/mobile/library/common/extensions/filters/http/assertion/config.cc b/mobile/library/common/extensions/filters/http/assertion/config.cc new file mode 100644 index 000000000000..bbb9061e414f --- /dev/null +++ b/mobile/library/common/extensions/filters/http/assertion/config.cc @@ -0,0 +1,29 @@ +#include "mobile/library/common/extensions/filters/http/assertion/config.h" + +#include "mobile/library/common/extensions/filters/http/assertion/filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Assertion { + +Http::FilterFactoryCb AssertionFilterFactory::createFilterFactoryFromProtoTyped( + const envoymobile::extensions::filters::http::assertion::Assertion& proto_config, + const std::string&, Server::Configuration::FactoryContext&) { + + AssertionFilterConfigSharedPtr filter_config = + std::make_shared(proto_config); + return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter(std::make_shared(filter_config)); + }; +} + +/** + * Static registration for the Assertion filter. @see NamedHttpFilterConfigFactory. + */ +REGISTER_FACTORY(AssertionFilterFactory, Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace Assertion +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/assertion/config.h b/mobile/library/common/extensions/filters/http/assertion/config.h new file mode 100644 index 000000000000..3c713fb37f82 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/assertion/config.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include "source/extensions/filters/http/common/factory_base.h" + +#include "mobile/library/common/extensions/filters/http/assertion/filter.pb.h" +#include "mobile/library/common/extensions/filters/http/assertion/filter.pb.validate.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Assertion { + +/** + * Config registration for the assertion filter. @see NamedHttpFilterConfigFactory. + */ +class AssertionFilterFactory + : public Common::FactoryBase { +public: + AssertionFilterFactory() : FactoryBase("assertion") {} + +private: + ::Envoy::Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoymobile::extensions::filters::http::assertion::Assertion& config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; +}; + +DECLARE_FACTORY(AssertionFilterFactory); + +} // namespace Assertion +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/assertion/filter.cc b/mobile/library/common/extensions/filters/http/assertion/filter.cc new file mode 100644 index 000000000000..9124ec38c625 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/assertion/filter.cc @@ -0,0 +1,224 @@ +#include "mobile/library/common/extensions/filters/http/assertion/filter.h" + +#include "envoy/http/codes.h" +#include "envoy/server/filter_config.h" + +#include "source/common/http/header_map_impl.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Assertion { + +AssertionFilterConfig::AssertionFilterConfig( + const envoymobile::extensions::filters::http::assertion::Assertion& proto_config) { + Common::Matcher::buildMatcher(proto_config.match_config(), matchers_); +} + +Extensions::Common::Matcher::Matcher& AssertionFilterConfig::rootMatcher() const { + ASSERT(!matchers_.empty()); + return *matchers_[0]; +} + +// Implementation of this filter is complicated by the fact that the streaming matchers have no +// explicit mechanism to handle end_stream. This means that we must infer that matching has failed +// if the stream ends with still-unsatisfied matches. We do this by potentially passing empty +// body data and empty trailers to the matchers in the event the stream ends without including +// these entities. +AssertionFilter::AssertionFilter(AssertionFilterConfigSharedPtr config) : config_(config) { + statuses_ = Extensions::Common::Matcher::Matcher::MatchStatusVector(config_->matchersSize()); + config_->rootMatcher().onNewStream(statuses_); +} + +Http::FilterHeadersStatus AssertionFilter::decodeHeaders(Http::RequestHeaderMap& headers, + bool end_stream) { + config_->rootMatcher().onHttpRequestHeaders(headers, statuses_); + auto& match_status = config_->rootMatcher().matchStatus(statuses_); + if (!match_status.matches_ && !match_status.might_change_status_) { + decoder_callbacks_->sendLocalReply(Http::Code::BadRequest, + "Request Headers do not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterHeadersStatus::StopIteration; + } + + if (end_stream) { + // Check if there are unsatisfied assertions about stream trailers. + auto empty_trailers = Http::RequestTrailerMapImpl::create(); + config_->rootMatcher().onHttpRequestTrailers(*empty_trailers, statuses_); + auto& final_match_status = config_->rootMatcher().matchStatus(statuses_); + if (!final_match_status.matches_ && !final_match_status.might_change_status_) { + decoder_callbacks_->sendLocalReply(Http::Code::BadRequest, + "Request Trailers do not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterHeadersStatus::StopIteration; + } + + // Because a stream only contains a single set of headers or trailers, if either fail to + // satisfy assertions, might_change_status_ will be false. Therefore if matches_ is still + // unsatisfied here, it must be because of body data. + if (!final_match_status.matches_) { + decoder_callbacks_->sendLocalReply(Http::Code::BadRequest, + "Request Body does not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterHeadersStatus::StopIteration; + } + } + + return Http::FilterHeadersStatus::Continue; +} + +Http::FilterDataStatus AssertionFilter::decodeData(Buffer::Instance& data, bool end_stream) { + config_->rootMatcher().onRequestBody(data, statuses_); + auto& match_status = config_->rootMatcher().matchStatus(statuses_); + if (!match_status.matches_ && !match_status.might_change_status_) { + decoder_callbacks_->sendLocalReply(Http::Code::BadRequest, + "Request Body does not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterDataStatus::StopIterationNoBuffer; + } + + if (end_stream) { + // Check if there are unsatisfied assertions about stream trailers. + auto empty_trailers = Http::RequestTrailerMapImpl::create(); + config_->rootMatcher().onHttpRequestTrailers(*empty_trailers, statuses_); + auto& match_status = config_->rootMatcher().matchStatus(statuses_); + if (!match_status.matches_ && !match_status.might_change_status_) { + decoder_callbacks_->sendLocalReply(Http::Code::BadRequest, + "Request Trailers do not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterDataStatus::StopIterationNoBuffer; + } + + // Because a stream only contains a single set of headers or trailers, if either fail to + // satisfy assertions, might_change_status_ will be false. Therefore if matches_ is still + // unsatisfied here, it must be because of body data. + if (!match_status.matches_) { + decoder_callbacks_->sendLocalReply(Http::Code::BadRequest, + "Request Body does not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterDataStatus::StopIterationNoBuffer; + } + } + return Http::FilterDataStatus::Continue; +} + +Http::FilterTrailersStatus AssertionFilter::decodeTrailers(Http::RequestTrailerMap& trailers) { + config_->rootMatcher().onHttpRequestTrailers(trailers, statuses_); + auto& match_status = config_->rootMatcher().matchStatus(statuses_); + if (!match_status.matches_ && !match_status.might_change_status_) { + decoder_callbacks_->sendLocalReply(Http::Code::BadRequest, + "Request Trailers do not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterTrailersStatus::StopIteration; + } + + // Because a stream only contains a single set of headers or trailers, if either fail to + // satisfy assertions, might_change_status_ will be false. Therefore if matches_ is still + // unsatisfied here, it must be because of body data. + if (!match_status.matches_) { + decoder_callbacks_->sendLocalReply(Http::Code::BadRequest, + "Request Body does not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterTrailersStatus::StopIteration; + } + return Http::FilterTrailersStatus::Continue; +} + +Http::FilterHeadersStatus AssertionFilter::encodeHeaders(Http::ResponseHeaderMap& headers, + bool end_stream) { + config_->rootMatcher().onHttpResponseHeaders(headers, statuses_); + auto& match_status = config_->rootMatcher().matchStatus(statuses_); + if (!match_status.matches_ && !match_status.might_change_status_) { + decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, + "Response Headers do not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterHeadersStatus::StopIteration; + } + + if (end_stream) { + // Check if there are unsatisfied assertions about stream trailers. + auto empty_trailers = Http::ResponseTrailerMapImpl::create(); + config_->rootMatcher().onHttpResponseTrailers(*empty_trailers, statuses_); + auto& final_match_status = config_->rootMatcher().matchStatus(statuses_); + if (!final_match_status.matches_ && !final_match_status.might_change_status_) { + decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, + "Response Trailers do not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterHeadersStatus::StopIteration; + } + + // Because a stream only contains a single set of headers or trailers, if either fail to + // satisfy assertions, might_change_status_ will be false. Therefore if matches_ is still + // unsatisfied here, it must be because of body data. + if (!final_match_status.matches_) { + decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, + "Response Body does not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterHeadersStatus::StopIteration; + } + } + + return Http::FilterHeadersStatus::Continue; +} + +Http::FilterDataStatus AssertionFilter::encodeData(Buffer::Instance& data, bool end_stream) { + config_->rootMatcher().onResponseBody(data, statuses_); + auto& match_status = config_->rootMatcher().matchStatus(statuses_); + if (!match_status.matches_ && !match_status.might_change_status_) { + decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, + "Response Body does not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterDataStatus::StopIterationNoBuffer; + } + + if (end_stream) { + // Check if there are unsatisfied assertions about stream trailers. + auto empty_trailers = Http::ResponseTrailerMapImpl::create(); + config_->rootMatcher().onHttpResponseTrailers(*empty_trailers, statuses_); + auto& match_status = config_->rootMatcher().matchStatus(statuses_); + if (!match_status.matches_ && !match_status.might_change_status_) { + decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, + "Response Trailers do not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterDataStatus::StopIterationNoBuffer; + } + + // Because a stream only contains a single set of headers or trailers, if either fail to + // satisfy assertions, might_change_status_ will be false. Therefore if matches_ is still + // unsatisfied here, it must be because of body data. + if (!match_status.matches_) { + decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, + "Response Body does not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterDataStatus::StopIterationNoBuffer; + } + } + return Http::FilterDataStatus::Continue; +} + +Http::FilterTrailersStatus AssertionFilter::encodeTrailers(Http::ResponseTrailerMap& trailers) { + config_->rootMatcher().onHttpResponseTrailers(trailers, statuses_); + auto& match_status = config_->rootMatcher().matchStatus(statuses_); + if (!match_status.matches_ && !match_status.might_change_status_) { + decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, + "Response Trailers do not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterTrailersStatus::StopIteration; + } + + // Because a stream only contains a single set of headers or trailers, if either fail to + // satisfy assertions, might_change_status_ will be false. Therefore if matches_ is still + // unsatisfied here, it must be because of body data. + if (!match_status.matches_) { + decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, + "Response Body does not match configured expectations", + nullptr, absl::nullopt, ""); + return Http::FilterTrailersStatus::StopIteration; + } + return Http::FilterTrailersStatus::Continue; +} + +} // namespace Assertion +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/assertion/filter.h b/mobile/library/common/extensions/filters/http/assertion/filter.h new file mode 100644 index 000000000000..bf68a3ccecc7 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/assertion/filter.h @@ -0,0 +1,56 @@ +#pragma once + +#include "envoy/http/filter.h" + +#include "source/extensions/common/matcher/matcher.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" + +#include "mobile/library/common/extensions/filters/http/assertion/filter.pb.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Assertion { + +class AssertionFilterConfig { +public: + AssertionFilterConfig( + const envoymobile::extensions::filters::http::assertion::Assertion& proto_config); + + Extensions::Common::Matcher::Matcher& rootMatcher() const; + size_t matchersSize() const { return matchers_.size(); } + +private: + std::vector matchers_; +}; + +using AssertionFilterConfigSharedPtr = std::shared_ptr; + +/** + * Filter to assert expectations on HTTP requests. + */ +class AssertionFilter final : public Http::PassThroughFilter { +public: + AssertionFilter(AssertionFilterConfigSharedPtr config); + + // StreamDecoderFilter + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers, + bool end_stream) override; + Http::FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override; + Http::FilterTrailersStatus decodeTrailers(Http::RequestTrailerMap& trailers) override; + + // StreamEncoderFilter + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap& headers, + bool end_stream) override; + Http::FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) override; + Http::FilterTrailersStatus encodeTrailers(Http::ResponseTrailerMap& trailers) override; + +private: + const AssertionFilterConfigSharedPtr config_; + Extensions::Common::Matcher::Matcher::MatchStatusVector statuses_; +}; + +} // namespace Assertion +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/assertion/filter.proto b/mobile/library/common/extensions/filters/http/assertion/filter.proto new file mode 100644 index 000000000000..fa6754a13bdc --- /dev/null +++ b/mobile/library/common/extensions/filters/http/assertion/filter.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package envoymobile.extensions.filters.http.assertion; + +import "envoy/config/common/matcher/v3/matcher.proto"; + +import "validate/validate.proto"; + +message Assertion { + // The match configuration. If the configuration matches the request frames the filter will send + // a local reply with Http::Code::OK on the last frame of the request stream (continuing on + // intervening frames). Otherwise, it will send a local reply with Http::Code::BadRequest. + // TODO: update upstream MatchPredicate proto to require at least one matcher. + envoy.config.common.matcher.v3.MatchPredicate match_config = 1 + [(validate.rules).message = {required: true}]; +} diff --git a/mobile/test/common/extensions/filters/http/assertion/BUILD b/mobile/test/common/extensions/filters/http/assertion/BUILD new file mode 100644 index 000000000000..b6200343cfcd --- /dev/null +++ b/mobile/test/common/extensions/filters/http/assertion/BUILD @@ -0,0 +1,21 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_mobile_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_mobile_package() + +envoy_cc_test( + name = "assertion_filter_test", + srcs = ["assertion_filter_test.cc"], + deps = [ + "//mobile/library/common/extensions/filters/http/assertion:config", + "//mobile/library/common/extensions/filters/http/assertion:filter_cc_proto", + "//test/mocks/http:http_mocks", + "//test/mocks/server:factory_context_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/mobile/test/common/extensions/filters/http/assertion/assertion_filter_test.cc b/mobile/test/common/extensions/filters/http/assertion/assertion_filter_test.cc new file mode 100644 index 000000000000..49b26dd2207a --- /dev/null +++ b/mobile/test/common/extensions/filters/http/assertion/assertion_filter_test.cc @@ -0,0 +1,562 @@ +#include "test/mocks/http/mocks.h" +#include "test/mocks/server/factory_context.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" +#include "mobile/library/common/extensions/filters/http/assertion/config.h" +#include "mobile/library/common/extensions/filters/http/assertion/filter.h" +#include "mobile/library/common/extensions/filters/http/assertion/filter.pb.h" + +using testing::ByMove; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Assertion { +namespace { + +class AssertionFilterTest : public testing::Test { +public: + void setUpFilter(std::string&& yaml) { + envoymobile::extensions::filters::http::assertion::Assertion config; + TestUtility::loadFromYaml(yaml, config); + config_ = std::make_shared(config); + filter_ = std::make_unique(config_); + filter_->setDecoderFilterCallbacks(decoder_callbacks_); + filter_->setEncoderFilterCallbacks(encoder_callbacks_); + } + + AssertionFilterConfigSharedPtr config_{}; + std::unique_ptr filter_{}; + NiceMock decoder_callbacks_; + NiceMock encoder_callbacks_; +}; + +TEST_F(AssertionFilterTest, RequestHeadersMatchWithEndStream) { + setUpFilter(R"EOF( +match_config: + http_request_headers_match: + headers: + - name: ":authority" + exact_match: test.code +)EOF"); + + Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true)); +} + +TEST_F(AssertionFilterTest, RequestHeadersMatch) { + setUpFilter(R"EOF( +match_config: + http_request_headers_match: + headers: + - name: ":authority" + exact_match: test.code +)EOF"); + + Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +} + +TEST_F(AssertionFilterTest, RequestHeadersNoMatchWithEndStream) { + setUpFilter(R"EOF( +match_config: + http_request_headers_match: + headers: + - name: ":authority" + exact_match: test.code +)EOF"); + + Http::TestRequestHeaderMapImpl request_headers{{":authority", "no.match"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::BadRequest, + "Request Headers do not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers, true)); +} + +TEST_F(AssertionFilterTest, RequestHeadersNoMatch) { + setUpFilter(R"EOF( +match_config: + http_request_headers_match: + headers: + - name: ":authority" + exact_match: test.code +)EOF"); + + Http::TestRequestHeaderMapImpl request_headers{{":authority", "no.match"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::BadRequest, + "Request Headers do not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers, false)); +} + +TEST_F(AssertionFilterTest, RequestHeadersMatchWithEndstreamAndDataMissing) { + setUpFilter(R"EOF( +match_config: + and_match: + rules: + - http_request_headers_match: + headers: + - name: ":authority" + exact_match: test.code + - http_request_generic_body_match: + patterns: + - string_match: match_me +)EOF"); + + Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::BadRequest, + "Request Body does not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers, true)); +} + +TEST_F(AssertionFilterTest, RequestHeadersMatchWithEndstreamAndTrailersMissing) { + setUpFilter(R"EOF( +match_config: + and_match: + rules: + - http_request_headers_match: + headers: + - name: ":authority" + exact_match: test.code + - http_request_trailers_match: + headers: + - name: "test-trailer" + exact_match: test.code +)EOF"); + + Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::BadRequest, + "Request Trailers do not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers, true)); +} + +TEST_F(AssertionFilterTest, RequestDataMatchWithEndStream) { + setUpFilter(R"EOF( +match_config: + http_request_generic_body_match: + patterns: + - string_match: match_me +)EOF"); + + Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(*body, true)); +} + +TEST_F(AssertionFilterTest, RequestDataMatch) { + setUpFilter(R"EOF( +match_config: + http_request_generic_body_match: + patterns: + - string_match: match_me +)EOF"); + + Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(*body, false)); +} + +TEST_F(AssertionFilterTest, RequestDataNoMatchFastPath) { + setUpFilter(R"EOF( +match_config: + http_request_generic_body_match: + bytes_limit: 1 + patterns: + - string_match: match_me +)EOF"); + + Buffer::InstancePtr body{new Buffer::OwnedImpl("garbage")}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::BadRequest, + "Request Body does not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(*body, false)); +} + +TEST_F(AssertionFilterTest, RequestDataNoMatchWithEndStream) { + setUpFilter(R"EOF( +match_config: + http_request_generic_body_match: + patterns: + - string_match: match_me +)EOF"); + + Buffer::InstancePtr body{new Buffer::OwnedImpl("garbage")}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::BadRequest, + "Request Body does not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(*body, true)); +} + +TEST_F(AssertionFilterTest, RequestDataMatchWithEndStreamAndTrailersMissing) { + setUpFilter(R"EOF( +match_config: + and_match: + rules: + - http_request_generic_body_match: + patterns: + - string_match: match_me + - http_request_trailers_match: + headers: + - name: "test-trailer" + exact_match: test.code +)EOF"); + + Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::BadRequest, + "Request Trailers do not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(*body, true)); +} + +TEST_F(AssertionFilterTest, RequestDataNoMatchAfterTrailers) { + setUpFilter(R"EOF( +match_config: + and_match: + rules: + - http_request_headers_match: + headers: + - name: ":authority" + exact_match: test.code + - http_request_generic_body_match: + patterns: + - string_match: match_me +)EOF"); + + Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; + Buffer::InstancePtr body{new Buffer::OwnedImpl("garbage")}; + Http::TestRequestTrailerMapImpl request_trailers{{"test-trailer", "test.code"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::BadRequest, + "Request Body does not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(*body, false)); + EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->decodeTrailers(request_trailers)); +} + +TEST_F(AssertionFilterTest, RequestTrailersMatch) { + setUpFilter(R"EOF( +match_config: + http_request_trailers_match: + headers: + - name: "test-trailer" + exact_match: test.code +)EOF"); + + Http::TestRequestTrailerMapImpl request_trailers{{"test-trailer", "test.code"}}; + + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +TEST_F(AssertionFilterTest, RequestTrailersNoMatch) { + setUpFilter(R"EOF( +match_config: + http_request_trailers_match: + headers: + - name: "test-trailer" + exact_match: test.code +)EOF"); + + Http::TestRequestTrailerMapImpl request_trailers{{"test-trailer", "no.match"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::BadRequest, + "Request Trailers do not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->decodeTrailers(request_trailers)); +} + +TEST_F(AssertionFilterTest, ResponseHeadersMatchWithEndStream) { + setUpFilter(R"EOF( +match_config: + http_response_headers_match: + headers: + - name: ":status" + exact_match: test.code +)EOF"); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers, true)); +} + +TEST_F(AssertionFilterTest, ResponseHeadersMatch) { + setUpFilter(R"EOF( +match_config: + http_response_headers_match: + headers: + - name: ":status" + exact_match: test.code +)EOF"); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers, false)); +} + +TEST_F(AssertionFilterTest, ResponseHeadersNoMatchWithEndStream) { + setUpFilter(R"EOF( +match_config: + http_response_headers_match: + headers: + - name: ":status" + exact_match: test.code +)EOF"); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "no.match"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::InternalServerError, + "Response Headers do not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->encodeHeaders(response_headers, true)); +} + +TEST_F(AssertionFilterTest, ResponseHeadersNoMatch) { + setUpFilter(R"EOF( +match_config: + http_response_headers_match: + headers: + - name: ":status" + exact_match: test.code +)EOF"); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "no.match"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::InternalServerError, + "Response Headers do not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->encodeHeaders(response_headers, false)); +} + +TEST_F(AssertionFilterTest, ResponseHeadersMatchWithEndstreamAndDataMissing) { + setUpFilter(R"EOF( +match_config: + and_match: + rules: + - http_response_headers_match: + headers: + - name: ":status" + exact_match: test.code + - http_response_generic_body_match: + patterns: + - string_match: match_me +)EOF"); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::InternalServerError, + "Response Body does not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->encodeHeaders(response_headers, true)); +} + +TEST_F(AssertionFilterTest, ResponseHeadersMatchWithEndstreamAndTrailersMissing) { + setUpFilter(R"EOF( +match_config: + and_match: + rules: + - http_response_headers_match: + headers: + - name: ":status" + exact_match: test.code + - http_response_trailers_match: + headers: + - name: "test-trailer" + exact_match: test.code +)EOF"); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::InternalServerError, + "Response Trailers do not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->encodeHeaders(response_headers, true)); +} + +TEST_F(AssertionFilterTest, ResponseDataMatchWithEndStream) { + setUpFilter(R"EOF( +match_config: + http_response_generic_body_match: + patterns: + - string_match: match_me +)EOF"); + + Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(*body, true)); +} + +TEST_F(AssertionFilterTest, ResponseDataMatch) { + setUpFilter(R"EOF( +match_config: + http_response_generic_body_match: + patterns: + - string_match: match_me +)EOF"); + + Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(*body, false)); +} + +TEST_F(AssertionFilterTest, ResponseDataNoMatchFastPath) { + setUpFilter(R"EOF( +match_config: + http_response_generic_body_match: + bytes_limit: 1 + patterns: + - string_match: match_me +)EOF"); + + Buffer::InstancePtr body{new Buffer::OwnedImpl("garbage")}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::InternalServerError, + "Response Body does not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(*body, false)); +} + +TEST_F(AssertionFilterTest, ResponseDataNoMatchWithEndStream) { + setUpFilter(R"EOF( +match_config: + http_response_generic_body_match: + patterns: + - string_match: match_me +)EOF"); + + Buffer::InstancePtr body{new Buffer::OwnedImpl("garbage")}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::InternalServerError, + "Response Body does not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(*body, true)); +} + +TEST_F(AssertionFilterTest, ResponseDataMatchWithEndStreamAndTrailersMissing) { + setUpFilter(R"EOF( +match_config: + and_match: + rules: + - http_response_generic_body_match: + patterns: + - string_match: match_me + - http_response_trailers_match: + headers: + - name: "test-trailer" + exact_match: test.code +)EOF"); + + Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::InternalServerError, + "Response Trailers do not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(*body, true)); +} + +TEST_F(AssertionFilterTest, ResponseDataNoMatchAfterTrailers) { + setUpFilter(R"EOF( +match_config: + and_match: + rules: + - http_response_headers_match: + headers: + - name: ":status" + exact_match: test.code + - http_response_generic_body_match: + patterns: + - string_match: match_me +)EOF"); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; + Buffer::InstancePtr body{new Buffer::OwnedImpl("garbage")}; + Http::TestResponseTrailerMapImpl response_trailers{{"test-trailer", "test.code"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::InternalServerError, + "Response Body does not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers, false)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(*body, false)); + EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->encodeTrailers(response_trailers)); +} + +TEST_F(AssertionFilterTest, ResponseTrailersMatch) { + setUpFilter(R"EOF( +match_config: + http_response_trailers_match: + headers: + - name: "test-trailer" + exact_match: test.code +)EOF"); + + Http::TestResponseTrailerMapImpl response_trailers{{"test-trailer", "test.code"}}; + + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers)); +} + +TEST_F(AssertionFilterTest, ResponseTrailersNoMatch) { + setUpFilter(R"EOF( +match_config: + http_response_trailers_match: + headers: + - name: "test-trailer" + exact_match: test.code +)EOF"); + + Http::TestResponseTrailerMapImpl response_trailers{{"test-trailer", "no.match"}}; + + EXPECT_CALL(decoder_callbacks_, + sendLocalReply(Http::Code::InternalServerError, + "Response Trailers do not match configured expectations", _, _, "")); + EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->encodeTrailers(response_trailers)); +} + +TEST(AssertionFilterFactoryTest, DEPRECATED_FEATURE_TEST(Config)) { + AssertionFilterFactory factory; + NiceMock context; + + envoymobile::extensions::filters::http::assertion::Assertion proto_config = + *dynamic_cast( + factory.createEmptyConfigProto().get()); + + std::string config_str = R"EOF( +match_config: + http_response_trailers_match: + headers: + - name: "test-trailer" + exact_match: test.code +)EOF"; + + TestUtility::loadFromYamlAndValidate(config_str, proto_config); + + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "test", context); + Http::MockFilterChainFactoryCallbacks filter_callbacks; + EXPECT_CALL(filter_callbacks, addStreamFilter(_)); + cb(filter_callbacks); +} + +} // namespace +} // namespace Assertion +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 544329f8e327..85c17f4967c3 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -374,8 +374,8 @@ EXTENSIONS = { # These can be changed to ["//visibility:public"], for downstream builds which # need to directly reference Envoy extensions. -EXTENSION_CONFIG_VISIBILITY = ["//:extension_config", "//:contrib_library", "//:examples_library"] -EXTENSION_PACKAGE_VISIBILITY = ["//:extension_library", "//:contrib_library", "//:examples_library"] +EXTENSION_CONFIG_VISIBILITY = ["//:extension_config", "//:contrib_library", "//:examples_library", "//:mobile_library"] +EXTENSION_PACKAGE_VISIBILITY = ["//:extension_library", "//:contrib_library", "//:examples_library", "//:mobile_library"] CONTRIB_EXTENSION_PACKAGE_VISIBILITY = ["//:contrib_library"] # Set this variable to true to disable alwayslink for envoy_cc_library. diff --git a/tools/code_format/envoy_build_fixer.py b/tools/code_format/envoy_build_fixer.py index 6a1146dec3fc..1ac976287623 100755 --- a/tools/code_format/envoy_build_fixer.py +++ b/tools/code_format/envoy_build_fixer.py @@ -35,6 +35,7 @@ PACKAGE_LOAD_BLOCK_REGEX = re.compile('("envoy_package".*?\)\n)', re.DOTALL) EXTENSION_PACKAGE_LOAD_BLOCK_REGEX = re.compile('("envoy_extension_package".*?\)\n)', re.DOTALL) CONTRIB_PACKAGE_LOAD_BLOCK_REGEX = re.compile('("envoy_contrib_package".*?\)\n)', re.DOTALL) +MOBILE_PACKAGE_LOAD_BLOCK_REGEX = re.compile('("envoy_mobile_package".*?\)\n)', re.DOTALL) # Match Buildozer 'print' output. Example of Buildozer print output: # cc_library json_transcoder_filter_lib [json_transcoder_filter.cc] (missing) (missing) @@ -84,6 +85,10 @@ def fix_package_and_license(path, contents): regex_to_use = CONTRIB_PACKAGE_LOAD_BLOCK_REGEX package_string = 'envoy_contrib_package' + if 'mobile/' in path: + regex_to_use = MOBILE_PACKAGE_LOAD_BLOCK_REGEX + package_string = 'envoy_mobile_package' + # Ensure we have an envoy_package import load if this is a real Envoy package. We also allow # the prefix to be overridden if envoy is included in a larger workspace. if re.search(ENVOY_RULE_REGEX, contents): From c5199512e9d3d0c393dfdfbb815f5796b1c3940b Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 2 Nov 2022 18:03:15 +0100 Subject: [PATCH 084/112] [Overload] Active downstream connections resource monitor (#23361) * [Overload] Active downstream connections resource monitor Signed-off-by: Kateryna Nezdolii --- CODEOWNERS | 1 + api/BUILD | 1 + .../downstream_connections/v3/BUILD | 12 +++ .../v3/downstream_connections.proto | 25 +++++ api/versioning/BUILD | 1 + source/extensions/extensions_build_config.bzl | 1 + source/extensions/extensions_metadata.yaml | 7 ++ .../resource_monitors/common/factory_base.h | 29 ++++++ .../downstream_connections/BUILD | 35 +++++++ .../downstream_connections/config.cc | 33 +++++++ .../downstream_connections/config.h | 32 ++++++ .../downstream_connections_monitor.cc | 67 +++++++++++++ .../downstream_connections_monitor.h | 38 +++++++ .../downstream_connections/BUILD | 38 +++++++ .../downstream_connections/config_test.cc | 85 ++++++++++++++++ .../downstream_connections_monitor_test.cc | 98 +++++++++++++++++++ 16 files changed, 503 insertions(+) create mode 100644 api/envoy/extensions/resource_monitors/downstream_connections/v3/BUILD create mode 100644 api/envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.proto create mode 100644 source/extensions/resource_monitors/downstream_connections/BUILD create mode 100644 source/extensions/resource_monitors/downstream_connections/config.cc create mode 100644 source/extensions/resource_monitors/downstream_connections/config.h create mode 100644 source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.cc create mode 100644 source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.h create mode 100644 test/extensions/resource_monitors/downstream_connections/BUILD create mode 100644 test/extensions/resource_monitors/downstream_connections/config_test.cc create mode 100644 test/extensions/resource_monitors/downstream_connections/downstream_connections_monitor_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index c5e135791b8d..eb137409a039 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -128,6 +128,7 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/resource_monitors/injected_resource @eziskind @htuch /*/extensions/resource_monitors/common @eziskind @htuch /*/extensions/resource_monitors/fixed_heap @eziskind @htuch +/*/extensions/resource_monitors/downstream_connections @antoniovicente @nezdolik @mattklein123 /*/extensions/retry/priority @snowp @alyssawilk /*/extensions/retry/priority/previous_priorities @snowp @alyssawilk /*/extensions/retry/host @snowp @alyssawilk diff --git a/api/BUILD b/api/BUILD index 29414b11265b..71cce936d5fe 100644 --- a/api/BUILD +++ b/api/BUILD @@ -257,6 +257,7 @@ proto_library( "//envoy/extensions/rbac/matchers/upstream_ip_port/v3:pkg", "//envoy/extensions/regex_engines/v3:pkg", "//envoy/extensions/request_id/uuid/v3:pkg", + "//envoy/extensions/resource_monitors/downstream_connections/v3:pkg", "//envoy/extensions/resource_monitors/fixed_heap/v3:pkg", "//envoy/extensions/resource_monitors/injected_resource/v3:pkg", "//envoy/extensions/retry/host/omit_canary_hosts/v3:pkg", diff --git a/api/envoy/extensions/resource_monitors/downstream_connections/v3/BUILD b/api/envoy/extensions/resource_monitors/downstream_connections/v3/BUILD new file mode 100644 index 000000000000..ec1e778e06e5 --- /dev/null +++ b/api/envoy/extensions/resource_monitors/downstream_connections/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_udpa//xds/annotations/v3:pkg", + ], +) diff --git a/api/envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.proto b/api/envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.proto new file mode 100644 index 000000000000..782acda56b4a --- /dev/null +++ b/api/envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package envoy.extensions.resource_monitors.downstream_connections.v3; + +import "xds/annotations/v3/status.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.resource_monitors.downstream_connections.v3"; +option java_outer_classname = "DownstreamConnectionsProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/resource_monitors/downstream_connections/v3;downstream_connectionsv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; +option (xds.annotations.v3.file_status).work_in_progress = true; + +// [#protodoc-title: Downstream connections] +// [#extension: envoy.resource_monitors.downstream_connections] + +// The downstream connections resource monitor tracks the global number of open downstream connections. +message DownstreamConnectionsConfig { + // Maximum threshold for global open downstream connections, defaults to 0. + // If monitor is enabled in Overload manager api, this field should be explicitly configured with value greater than 0. + int64 max_active_downstream_connections = 1 [(validate.rules).int64 = {gt: 0}]; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index a3935c83ab22..ea1ae783ab1c 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -202,6 +202,7 @@ proto_library( "//envoy/extensions/rbac/matchers/upstream_ip_port/v3:pkg", "//envoy/extensions/regex_engines/v3:pkg", "//envoy/extensions/request_id/uuid/v3:pkg", + "//envoy/extensions/resource_monitors/downstream_connections/v3:pkg", "//envoy/extensions/resource_monitors/fixed_heap/v3:pkg", "//envoy/extensions/resource_monitors/injected_resource/v3:pkg", "//envoy/extensions/retry/host/omit_canary_hosts/v3:pkg", diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 85c17f4967c3..b5c964cdb6fe 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -169,6 +169,7 @@ EXTENSIONS = { "envoy.resource_monitors.fixed_heap": "//source/extensions/resource_monitors/fixed_heap:config", "envoy.resource_monitors.injected_resource": "//source/extensions/resource_monitors/injected_resource:config", + "envoy.resource_monitors.downstream_connections": "//source/extensions/resource_monitors/downstream_connections:config", # # Stat sinks diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index e6a548b2f629..a537ef2a9f4b 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -847,6 +847,13 @@ envoy.request_id.uuid: status: stable type_urls: - envoy.extensions.request_id.uuid.v3.UuidRequestIdConfig +envoy.resource_monitors.downstream_connections: + categories: + - envoy.resource_monitors + security_posture: data_plane_agnostic + status: wip + type_urls: + - envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig envoy.resource_monitors.fixed_heap: categories: - envoy.resource_monitors diff --git a/source/extensions/resource_monitors/common/factory_base.h b/source/extensions/resource_monitors/common/factory_base.h index 13bd2e86aa15..637a6d23102f 100644 --- a/source/extensions/resource_monitors/common/factory_base.h +++ b/source/extensions/resource_monitors/common/factory_base.h @@ -37,6 +37,35 @@ class FactoryBase : public Server::Configuration::ResourceMonitorFactory { const std::string name_; }; +template +class ProactiveFactoryBase : public Server::Configuration::ProactiveResourceMonitorFactory { +public: + Server::ProactiveResourceMonitorPtr createProactiveResourceMonitor( + const Protobuf::Message& config, + Server::Configuration::ResourceMonitorFactoryContext& context) override { + return createProactiveResourceMonitorFromProtoTyped( + MessageUtil::downcastAndValidate(config, + context.messageValidationVisitor()), + context); + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return name_; } + +protected: + ProactiveFactoryBase(const std::string& name) : name_(name) {} + +private: + virtual Server::ProactiveResourceMonitorPtr createProactiveResourceMonitorFromProtoTyped( + const ConfigProto& config, + Server::Configuration::ResourceMonitorFactoryContext& context) PURE; + + const std::string name_; +}; + } // namespace Common } // namespace ResourceMonitors } // namespace Extensions diff --git a/source/extensions/resource_monitors/downstream_connections/BUILD b/source/extensions/resource_monitors/downstream_connections/BUILD new file mode 100644 index 000000000000..96c906d1653c --- /dev/null +++ b/source/extensions/resource_monitors/downstream_connections/BUILD @@ -0,0 +1,35 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "downstream_connections_monitor", + srcs = ["downstream_connections_monitor.cc"], + hdrs = ["downstream_connections_monitor.h"], + deps = [ + "//envoy/server:proactive_resource_monitor_interface", + "//envoy/server:resource_monitor_config_interface", + "//source/common/common:thread_synchronizer_lib", + "@envoy_api//envoy/extensions/resource_monitors/downstream_connections/v3:pkg_cc_proto", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":downstream_connections_monitor", + "//envoy/registry", + "//source/common/common:assert_lib", + "//source/extensions/resource_monitors/common:factory_base_lib", + "@envoy_api//envoy/extensions/resource_monitors/downstream_connections/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/resource_monitors/downstream_connections/config.cc b/source/extensions/resource_monitors/downstream_connections/config.cc new file mode 100644 index 000000000000..fbfd77497a53 --- /dev/null +++ b/source/extensions/resource_monitors/downstream_connections/config.cc @@ -0,0 +1,33 @@ +#include "source/extensions/resource_monitors/downstream_connections/config.h" + +#include "envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.pb.h" +#include "envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.pb.validate.h" +#include "envoy/registry/registry.h" + +#include "source/common/protobuf/utility.h" +#include "source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace DownstreamConnections { + +Server::ProactiveResourceMonitorPtr +ActiveDownstreamConnectionsMonitorFactory::createProactiveResourceMonitorFromProtoTyped( + const envoy::extensions::resource_monitors::downstream_connections::v3:: + DownstreamConnectionsConfig& config, + Server::Configuration::ResourceMonitorFactoryContext& /*unused_context*/) { + return std::make_unique(config); +} + +/** + * Static registration for the downstream connections resource monitor factory. @see + * RegistryFactory. + */ +REGISTER_FACTORY(ActiveDownstreamConnectionsMonitorFactory, + Server::Configuration::ProactiveResourceMonitorFactory); + +} // namespace DownstreamConnections +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/resource_monitors/downstream_connections/config.h b/source/extensions/resource_monitors/downstream_connections/config.h new file mode 100644 index 000000000000..d71b37ab1a7f --- /dev/null +++ b/source/extensions/resource_monitors/downstream_connections/config.h @@ -0,0 +1,32 @@ +#pragma once + +#include "envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.pb.h" +#include "envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.pb.validate.h" +#include "envoy/server/resource_monitor_config.h" + +#include "source/extensions/resource_monitors/common/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace DownstreamConnections { + +class ActiveDownstreamConnectionsMonitorFactory + : public Common::ProactiveFactoryBase< + envoy::extensions::resource_monitors::downstream_connections::v3:: + DownstreamConnectionsConfig> { +public: + ActiveDownstreamConnectionsMonitorFactory() + : ProactiveFactoryBase("envoy.resource_monitors.downstream_connections") {} + +private: + Server::ProactiveResourceMonitorPtr createProactiveResourceMonitorFromProtoTyped( + const envoy::extensions::resource_monitors::downstream_connections::v3:: + DownstreamConnectionsConfig& config, + Server::Configuration::ResourceMonitorFactoryContext& context) override; +}; + +} // namespace DownstreamConnections +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.cc b/source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.cc new file mode 100644 index 000000000000..62467283e867 --- /dev/null +++ b/source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.cc @@ -0,0 +1,67 @@ +#include "source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.h" + +#include "envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.pb.h" + +#include "source/common/common/assert.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace DownstreamConnections { + +ActiveDownstreamConnectionsResourceMonitor::ActiveDownstreamConnectionsResourceMonitor( + const envoy::extensions::resource_monitors::downstream_connections::v3:: + DownstreamConnectionsConfig& config) + : max_(config.max_active_downstream_connections()), current_(0){}; + +bool ActiveDownstreamConnectionsResourceMonitor::tryAllocateResource(int64_t increment) { + // No synchronization is imposed on other reads or writes. + auto current = current_.load(std::memory_order_relaxed); + while (current + increment <= max_) { + // Testing hook. + synchronizer_.syncPoint("try_allocate_pre_cas"); + // `current_` value is atomically compared to `current`. + // In case values are bitwise equal, `current` value is replaced with `current + increment`. + // The write will be visible to other threads accessing `current_`. + // If `current` is not equal to value stored in `current_`, `current` will be reloaded with the + // latest value of `current_` and no synchronization will be imposed on other reads or writes. + // After value reload next loop iteration will be attempted until suggested increment breaches + // `max_` or cas operation is successful. + if (current_.compare_exchange_weak(current, current + increment, std::memory_order_release, + std::memory_order_relaxed)) { + return true; + } + } + return false; +} + +bool ActiveDownstreamConnectionsResourceMonitor::tryDeallocateResource(int64_t decrement) { + // No synchronization is imposed on other reads or writes. + auto current = current_.load(std::memory_order_relaxed); + while (current - decrement >= 0) { + // Testing hook. + synchronizer_.syncPoint("try_deallocate_pre_cas"); + // `current_` value is atomically compared to `current`. + // In case values are bitwise equal, `current` value is replaced with `current - increment`. + // The write will be visible to other threads accessing `current_`. + // If `current` is not equal to value stored in `current_`, `current` will be reloaded with the + // latest value of `current_` and no synchronization will be imposed on other reads or writes. + // After value reload next loop iteration will be attempted until suggested decrement goes below + // 0 or cas operation is successful. + if (current_.compare_exchange_weak(current, current - decrement, std::memory_order_release, + std::memory_order_relaxed)) { + return true; + } + } + return false; +} + +int64_t ActiveDownstreamConnectionsResourceMonitor::currentResourceUsage() const { + return current_.load(); +} +int64_t ActiveDownstreamConnectionsResourceMonitor::maxResourceUsage() const { return max_; }; + +} // namespace DownstreamConnections +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.h b/source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.h new file mode 100644 index 000000000000..21d35bacaab5 --- /dev/null +++ b/source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.h @@ -0,0 +1,38 @@ +#pragma once + +#include "envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.pb.h" +#include "envoy/server/proactive_resource_monitor.h" + +#include "source/common/common/thread_synchronizer.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace DownstreamConnections { + +class ActiveDownstreamConnectionsResourceMonitor : public Server::ProactiveResourceMonitor { +public: + ActiveDownstreamConnectionsResourceMonitor( + const envoy::extensions::resource_monitors::downstream_connections::v3:: + DownstreamConnectionsConfig& config); + + bool tryAllocateResource(int64_t increment) override; + + bool tryDeallocateResource(int64_t decrement) override; + + int64_t currentResourceUsage() const override; + int64_t maxResourceUsage() const override; + +protected: + const int64_t max_; + std::atomic current_; + // Used for testing only. + mutable Thread::ThreadSynchronizer synchronizer_; + + friend class ActiveDownstreamConnectionsMonitorTest; +}; + +} // namespace DownstreamConnections +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/resource_monitors/downstream_connections/BUILD b/test/extensions/resource_monitors/downstream_connections/BUILD new file mode 100644 index 000000000000..58f2b7505b56 --- /dev/null +++ b/test/extensions/resource_monitors/downstream_connections/BUILD @@ -0,0 +1,38 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "downstream_connections_monitor_test", + srcs = ["downstream_connections_monitor_test.cc"], + extension_names = ["envoy.resource_monitors.downstream_connections"], + external_deps = ["abseil_optional"], + deps = [ + "//source/extensions/resource_monitors/downstream_connections:downstream_connections_monitor", + "@envoy_api//envoy/extensions/resource_monitors/downstream_connections/v3:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.resource_monitors.downstream_connections"], + deps = [ + "//envoy/registry", + "//source/extensions/resource_monitors/downstream_connections:config", + "//source/server:resource_monitor_config_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/server:options_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/resource_monitors/downstream_connections/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/resource_monitors/downstream_connections/config_test.cc b/test/extensions/resource_monitors/downstream_connections/config_test.cc new file mode 100644 index 000000000000..63b88b741681 --- /dev/null +++ b/test/extensions/resource_monitors/downstream_connections/config_test.cc @@ -0,0 +1,85 @@ +#include "envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.pb.h" +#include "envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.pb.validate.h" +#include "envoy/registry/registry.h" + +#include "source/extensions/resource_monitors/downstream_connections/config.h" +#include "source/server/resource_monitor_config_impl.h" + +#include "test/mocks/event/mocks.h" +#include "test/mocks/server/options.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace DownstreamConnections { +namespace { + +TEST(ActiveDownstreamConnectionsMonitorFactoryTest, CreateMonitorInvalidConfig) { + auto factory = + Registry::FactoryRegistry::getFactory( + "envoy.resource_monitors.downstream_connections"); + ASSERT_NE(factory, nullptr); + EXPECT_EQ("envoy.resource_monitors.downstream_connections", factory->name()); + + envoy::extensions::resource_monitors::downstream_connections::v3::DownstreamConnectionsConfig + config; + config.set_max_active_downstream_connections(-1); + Event::MockDispatcher dispatcher; + Api::ApiPtr api = Api::createApiForTest(); + Server::MockOptions options; + Server::Configuration::ResourceMonitorFactoryContextImpl context( + dispatcher, options, *api, ProtobufMessage::getStrictValidationVisitor()); + EXPECT_THROW_WITH_REGEX(factory->createProactiveResourceMonitor(config, context), + ProtoValidationException, + "Proto constraint validation failed " + "\\(DownstreamConnectionsConfigValidationError." + "MaxActiveDownstreamConnections: value must be greater than 0"); +} + +TEST(ActiveDownstreamConnectionsMonitorFactoryTest, CreateCustomMonitor) { + auto factory = + Registry::FactoryRegistry::getFactory( + "envoy.resource_monitors.downstream_connections"); + ASSERT_NE(factory, nullptr); + EXPECT_EQ("envoy.resource_monitors.downstream_connections", factory->name()); + + envoy::extensions::resource_monitors::downstream_connections::v3::DownstreamConnectionsConfig + config; + config.set_max_active_downstream_connections(1); + Event::MockDispatcher dispatcher; + Api::ApiPtr api = Api::createApiForTest(); + Server::MockOptions options; + Server::Configuration::ResourceMonitorFactoryContextImpl context( + dispatcher, options, *api, ProtobufMessage::getStrictValidationVisitor()); + auto monitor = factory->createProactiveResourceMonitor(config, context); + EXPECT_NE(monitor, nullptr); +} + +TEST(ActiveDownstreamConnectionsMonitorFactoryTest, CreateDefaultMonitor) { + auto factory = + Registry::FactoryRegistry::getFactory( + "envoy.resource_monitors.downstream_connections"); + ASSERT_NE(factory, nullptr); + + Event::MockDispatcher dispatcher; + Api::ApiPtr api = Api::createApiForTest(); + Server::MockOptions options; + Server::Configuration::ResourceMonitorFactoryContextImpl context( + dispatcher, options, *api, ProtobufMessage::getStrictValidationVisitor()); + auto config = factory->createEmptyConfigProto(); + + EXPECT_THROW_WITH_REGEX(factory->createProactiveResourceMonitor(*config, context), + ProtoValidationException, + "Proto constraint validation failed " + "\\(DownstreamConnectionsConfigValidationError." + "MaxActiveDownstreamConnections: value must be greater than 0"); +} + +} // namespace +} // namespace DownstreamConnections +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/resource_monitors/downstream_connections/downstream_connections_monitor_test.cc b/test/extensions/resource_monitors/downstream_connections/downstream_connections_monitor_test.cc new file mode 100644 index 000000000000..d276a4c53871 --- /dev/null +++ b/test/extensions/resource_monitors/downstream_connections/downstream_connections_monitor_test.cc @@ -0,0 +1,98 @@ +#include "envoy/extensions/resource_monitors/downstream_connections/v3/downstream_connections.pb.h" + +#include "source/extensions/resource_monitors/downstream_connections/downstream_connections_monitor.h" + +#include "absl/types/optional.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace ResourceMonitors { +namespace DownstreamConnections { + +class ActiveDownstreamConnectionsMonitorTest : public testing::Test { +public: + void initialize(const envoy::extensions::resource_monitors::downstream_connections::v3:: + DownstreamConnectionsConfig& config) { + monitor_ = std::make_unique(config); + } + + Thread::ThreadSynchronizer& synchronizer() { return monitor_->synchronizer_; } + + std::unique_ptr monitor_; +}; + +TEST_F(ActiveDownstreamConnectionsMonitorTest, CannotAllocateDeallocateResourceWithDefaultConfig) { + envoy::extensions::resource_monitors::downstream_connections::v3::DownstreamConnectionsConfig + config; + initialize(config); + EXPECT_FALSE(monitor_->tryAllocateResource(1)); + EXPECT_EQ(0, monitor_->currentResourceUsage()); +} + +TEST_F(ActiveDownstreamConnectionsMonitorTest, ComputesCorrectUsage) { + envoy::extensions::resource_monitors::downstream_connections::v3::DownstreamConnectionsConfig + config; + config.set_max_active_downstream_connections(10); + initialize(config); + EXPECT_EQ(0, monitor_->currentResourceUsage()); + EXPECT_EQ(10, monitor_->maxResourceUsage()); + EXPECT_TRUE(monitor_->tryAllocateResource(3)); + EXPECT_EQ(3, monitor_->currentResourceUsage()); + EXPECT_TRUE(monitor_->tryDeallocateResource(2)); + EXPECT_EQ(1, monitor_->currentResourceUsage()); +} + +TEST_F(ActiveDownstreamConnectionsMonitorTest, FailsToAllocateDeallocateWhenMinMaxHit) { + envoy::extensions::resource_monitors::downstream_connections::v3::DownstreamConnectionsConfig + config; + config.set_max_active_downstream_connections(1); + initialize(config); + EXPECT_EQ(0, monitor_->currentResourceUsage()); + EXPECT_EQ(1, monitor_->maxResourceUsage()); + EXPECT_TRUE(monitor_->tryAllocateResource(1)); + EXPECT_FALSE(monitor_->tryAllocateResource(1)); + EXPECT_TRUE(monitor_->tryDeallocateResource(1)); + EXPECT_EQ(0, monitor_->currentResourceUsage()); + EXPECT_FALSE(monitor_->tryDeallocateResource(1)); +} + +TEST_F(ActiveDownstreamConnectionsMonitorTest, AllocateCasMultithreaded) { + envoy::extensions::resource_monitors::downstream_connections::v3::DownstreamConnectionsConfig + config; + config.set_max_active_downstream_connections(1); + initialize(config); + synchronizer().enable(); + // Start a thread and wait pre-CAS. + synchronizer().waitOn("try_allocate_pre_cas"); + std::thread t1([&] { EXPECT_FALSE(monitor_->tryAllocateResource(1)); }); + // Wait until the thread is actually waiting. + synchronizer().barrierOn("try_allocate_pre_cas"); + // Increase connection counter to 1, which should cause the CAS to fail on the other thread. + EXPECT_TRUE(monitor_->tryAllocateResource(1)); + synchronizer().signal("try_allocate_pre_cas"); + t1.join(); +} + +TEST_F(ActiveDownstreamConnectionsMonitorTest, DeallocateCasMultithreaded) { + envoy::extensions::resource_monitors::downstream_connections::v3::DownstreamConnectionsConfig + config; + config.set_max_active_downstream_connections(3); + initialize(config); + synchronizer().enable(); + EXPECT_TRUE(monitor_->tryAllocateResource(3)); + // Start a thread and wait pre-CAS. + synchronizer().waitOn("try_deallocate_pre_cas"); + std::thread t1([&] { EXPECT_FALSE(monitor_->tryDeallocateResource(1)); }); + // Wait until the thread is actually waiting. + synchronizer().barrierOn("try_deallocate_pre_cas"); + EXPECT_TRUE(monitor_->tryDeallocateResource(3)); + synchronizer().signal("try_deallocate_pre_cas"); + t1.join(); +} + +} // namespace DownstreamConnections +} // namespace ResourceMonitors +} // namespace Extensions +} // namespace Envoy From 943f7201af95da94b2e79948a877ad2540a6ecf2 Mon Sep 17 00:00:00 2001 From: Felix Du Date: Thu, 3 Nov 2022 01:58:14 +0800 Subject: [PATCH 085/112] Fix sip_proxy build (#23798) Fixes #23693 Signed-off-by: Felix Du --- contrib/sip_proxy/filters/network/source/metadata.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/sip_proxy/filters/network/source/metadata.cc b/contrib/sip_proxy/filters/network/source/metadata.cc index b697feaea938..0cbab8b16958 100644 --- a/contrib/sip_proxy/filters/network/source/metadata.cc +++ b/contrib/sip_proxy/filters/network/source/metadata.cc @@ -26,8 +26,7 @@ void SipHeader::parseHeader() { header = header.substr(0, found); } // Has message Type in header - // Eg: Route: + // Eg: Route: if (std::size_t found = header.find(": "); found != absl::string_view::npos) { header = header.substr(found + 2); } From cfd126afe349804dda866c96abb7773ded5475fc Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 2 Nov 2022 14:11:20 -0400 Subject: [PATCH 086/112] typo: replace 'Enovy' with 'Envoy' (#23800) Signed-off-by: JP Simard --- source/common/runtime/runtime_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 7a13f274770e..d034a34664d8 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -382,7 +382,7 @@ void ProtoLayer::walkProtoValue(const ProtobufWkt::Value& v, const std::string& if (hasRuntimePrefix(prefix) && !isRuntimeFeature(prefix)) { IS_ENVOY_BUG(absl::StrCat( "Using a removed guard ", prefix, - ". In future version of Enovy this will be treated as invalid configuration")); + ". In future version of Envoy this will be treated as invalid configuration")); } values_.emplace(prefix, SnapshotImpl::createEntry(v)); break; From 36203ded7fc55b6ea0ab5d0e1ccb57a9846539de Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 2 Nov 2022 19:36:33 +0000 Subject: [PATCH 087/112] deps: Bump `rules_fuzzing` -> 0.3.2 (#23731) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index e7689a501413..86f4c1c7b37b 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -72,11 +72,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Fuzzing Rules for Bazel", project_desc = "Bazel rules for fuzz tests", project_url = "https://github.com/bazelbuild/rules_fuzzing", - version = "0.3.1", - sha256 = "4965ff7341f4759f07c83b146f603d6e8cfc35ef99fee3ef39bf61ffa96b1f8b", + version = "0.3.2", + sha256 = "f85dc70bb9672af0e350686461fe6fdd0d61e10e75645f9e44fedf549b21e369", strip_prefix = "rules_fuzzing-{version}", urls = ["https://github.com/bazelbuild/rules_fuzzing/archive/v{version}.tar.gz"], - release_date = "2022-01-24", + release_date = "2022-08-31", use_category = ["test_only"], implied_untracked_deps = [ # This is a repository rule generated to define an OSS-Fuzz fuzzing From 06f7cf8a9a76ee4fc2a3578a3b67152fa5ad0547 Mon Sep 17 00:00:00 2001 From: Xie Zhihao Date: Thu, 3 Nov 2022 04:25:38 +0800 Subject: [PATCH 088/112] test: wait for end stream in HTTP timeout integration tests (#23792) Signed-off-by: Xie Zhihao --- test/integration/http_timeout_integration_test.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/integration/http_timeout_integration_test.cc b/test/integration/http_timeout_integration_test.cc index 2d6b657cc64b..19fa1ffe8bd3 100644 --- a/test/integration/http_timeout_integration_test.cc +++ b/test/integration/http_timeout_integration_test.cc @@ -38,7 +38,7 @@ TEST_P(HttpTimeoutIntegrationTest, GlobalTimeout) { timeSystem().advanceTimeWait(std::chrono::milliseconds(501)); // Ensure we got a timeout downstream and canceled the upstream request. - response->waitForHeaders(); + ASSERT_TRUE(response->waitForEndStream()); ASSERT_TRUE(upstream_request_->waitForReset(std::chrono::seconds(15))); codec_client_->close(); @@ -81,7 +81,7 @@ TEST_P(HttpTimeoutIntegrationTest, UseTimeoutSetByEgressEnvoy) { timeSystem().advanceTimeWait(std::chrono::milliseconds(301)); // Ensure we got a timeout downstream and canceled the upstream request. - response->waitForHeaders(); + ASSERT_TRUE(response->waitForEndStream()); ASSERT_TRUE(upstream_request_->waitForReset(std::chrono::seconds(15))); codec_client_->close(); @@ -122,7 +122,7 @@ TEST_P(HttpTimeoutIntegrationTest, DeriveTimeoutInIngressEnvoy) { timeSystem().advanceTimeWait(std::chrono::milliseconds(501)); // Ensure we got a timeout downstream and canceled the upstream request. - response->waitForHeaders(); + ASSERT_TRUE(response->waitForEndStream()); ASSERT_TRUE(upstream_request_->waitForReset(std::chrono::seconds(15))); codec_client_->close(); @@ -164,7 +164,7 @@ TEST_P(HttpTimeoutIntegrationTest, IgnoreTimeoutSetByEgressEnvoy) { timeSystem().advanceTimeWait(std::chrono::milliseconds(501)); // Ensure we got a timeout downstream and canceled the upstream request. - response->waitForHeaders(); + ASSERT_TRUE(response->waitForEndStream()); ASSERT_TRUE(upstream_request_->waitForReset(std::chrono::seconds(15))); codec_client_->close(); @@ -261,7 +261,7 @@ TEST_P(HttpTimeoutIntegrationTest, PerTryTimeout) { // Trigger global timeout. timeSystem().advanceTimeWait(std::chrono::milliseconds(100)); - response->waitForHeaders(); + ASSERT_TRUE(response->waitForEndStream()); codec_client_->close(); @@ -313,7 +313,7 @@ TEST_P(HttpTimeoutIntegrationTest, PerTryTimeoutWithoutGlobalTimeout) { Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}}; upstream_request_->encodeHeaders(response_headers, true); - response->waitForHeaders(); + ASSERT_TRUE(response->waitForEndStream()); codec_client_->close(); EXPECT_TRUE(upstream_request_->complete()); @@ -368,7 +368,7 @@ TEST_P(HttpTimeoutIntegrationTest, HedgedPerTryTimeout) { Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}}; upstream_request_->encodeHeaders(response_headers, true); - response->waitForHeaders(); + ASSERT_TRUE(response->waitForEndStream()); // The second request should be reset since we used the response from the first request. ASSERT_TRUE(upstream_request2->waitForReset(std::chrono::seconds(15))); From 7ac5336bc807f32e84f04cd20d2c95d95e923aa7 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 2 Nov 2022 15:40:24 -0700 Subject: [PATCH 089/112] ci: use host docker gid for envoybuild (#23803) Previously `/var/run/docker.sock` is readable/writable inside docker run because group ID of `envoygroup` coincidentally matches host docker group, while it is no longer true during rolling out new image. Fixing that by forcing `envoygroup` has host docker group ID. Risk Level: Low Testing: CI Docs Changes: Release Notes: Platform Specific Features: Signed-off-by: Lizan Zhou --- ci/run_envoy_docker.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index de0e6012ac0d..cad4b993cd2f 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -41,8 +41,9 @@ else BUILD_DIR_MOUNT_DEST=/build SOURCE_DIR="${PWD}" SOURCE_DIR_MOUNT_DEST=/source - START_COMMAND=("/bin/bash" "-lc" "groupadd --gid $(id -g) -f envoygroup \ - && useradd -o --uid $(id -u) --gid $(id -g) --no-create-home --home-dir /build envoybuild \ + DOCKER_GID="$(stat -c '%g' /var/run/docker.sock)" + START_COMMAND=("/bin/bash" "-lc" "groupadd --gid ${DOCKER_GID} -f envoygroup \ + && useradd -o --uid $(id -u) --gid ${DOCKER_GID} --no-create-home --home-dir /build envoybuild \ && usermod -a -G pcap envoybuild \ && chown envoybuild:envoygroup /build \ && sudo -EHs -u envoybuild bash -c 'cd /source && $*'") From 8d523d4cc497eafb14d9400c7398a34d2dde7171 Mon Sep 17 00:00:00 2001 From: code Date: Thu, 3 Nov 2022 07:11:21 +0800 Subject: [PATCH 090/112] generic proxy: rds support for generic proxy (#23743) * generic proxy: rds support for generic proxy Signed-off-by: wbpcode * address comments Signed-off-by: wbpcode Signed-off-by: wbpcode --- .../generic_proxy/v3/generic_proxy.proto | 2 - changelogs/current.yaml | 3 + .../filters/network/source/BUILD | 24 +++++ .../filters/network/source/config.cc | 17 +++- .../filters/network/source/interface/BUILD | 1 + .../filters/network/source/interface/route.h | 5 +- .../filters/network/source/proxy.cc | 26 ++++- .../filters/network/source/proxy.h | 29 +++--- .../filters/network/source/rds.h | 24 +++++ .../filters/network/source/rds_impl.h | 26 +++++ .../filters/network/source/route.cc | 9 +- .../filters/network/source/route.h | 9 +- .../generic_proxy/filters/network/test/BUILD | 2 + .../filters/network/test/config_test.cc | 95 +++++++++++++++++++ .../filters/network/test/proxy_test.cc | 64 ++++--------- .../filters/network/test/route_test.cc | 2 +- 16 files changed, 264 insertions(+), 74 deletions(-) create mode 100644 contrib/generic_proxy/filters/network/source/rds.h create mode 100644 contrib/generic_proxy/filters/network/source/rds_impl.h diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.proto index 7ba257ce59be..bd2efca5ab19 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.proto +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.proto @@ -35,7 +35,6 @@ message GenericProxy { oneof route_specifier { option (validate.required) = true; - // [#not-implemented-hide:] // The generic proxies route table will be dynamically loaded via the meta RDS API. GenericRds generic_rds = 3; @@ -50,7 +49,6 @@ message GenericProxy { repeated config.core.v3.TypedExtensionConfig filters = 5; } -// [#not-implemented-hide:] message GenericRds { // Configuration source specifier for RDS. config.core.v3.ConfigSource config_source = 1 [(validate.rules).message = {required: true}]; diff --git a/changelogs/current.yaml b/changelogs/current.yaml index b253d5ecae2b..54de9a576177 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -77,6 +77,9 @@ new_features: - area: http change: | allowing the dynamic forward proxy cluster to :ref:`allow_coalesced_connections ` for HTTP/2 and HTTP/3 connections. +- area: generic_proxy + change: | + added :ref:`generic rds support `. - area: thrift_proxy change: | added ``envoy.reloadable_features.thrift_allow_negative_field_ids`` to support negative field ids for legacy thrift service. diff --git a/contrib/generic_proxy/filters/network/source/BUILD b/contrib/generic_proxy/filters/network/source/BUILD index dd3b53e7413a..5bab5f6bfebd 100644 --- a/contrib/generic_proxy/filters/network/source/BUILD +++ b/contrib/generic_proxy/filters/network/source/BUILD @@ -18,6 +18,7 @@ envoy_cc_library( "proxy.h", ], deps = [ + ":rds_lib", ":route_lib", "//contrib/generic_proxy/filters/network/source/interface:codec_interface", "//contrib/generic_proxy/filters/network/source/router:router_lib", @@ -87,3 +88,26 @@ envoy_cc_library( ], alwayslink = 1, ) + +envoy_cc_library( + name = "rds_interface", + hdrs = ["rds.h"], + deps = [ + "//envoy/rds:rds_interface", + "@envoy_api//contrib/envoy/extensions/filters/network/generic_proxy/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "rds_lib", + hdrs = [ + "rds_impl.h", + ], + deps = [ + ":rds_interface", + ":route_lib", + "//envoy/rds:rds_interface", + "//source/common/rds:rds_lib", + "@envoy_api//contrib/envoy/extensions/filters/network/generic_proxy/v3:pkg_cc_proto", + ], +) diff --git a/contrib/generic_proxy/filters/network/source/config.cc b/contrib/generic_proxy/filters/network/source/config.cc index 7eaa28983952..7c7676a64295 100644 --- a/contrib/generic_proxy/filters/network/source/config.cc +++ b/contrib/generic_proxy/filters/network/source/config.cc @@ -1,15 +1,28 @@ #include "contrib/generic_proxy/filters/network/source/config.h" +#include "contrib/generic_proxy/filters/network/source/rds.h" +#include "contrib/generic_proxy/filters/network/source/rds_impl.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { namespace GenericProxy { +SINGLETON_MANAGER_REGISTRATION(generic_route_config_provider_manager); + Envoy::Network::FilterFactoryCb Factory::createFilterFactoryFromProtoTyped(const ProxyConfig& proto_config, Envoy::Server::Configuration::FactoryContext& context) { - auto config = std::make_shared(proto_config, context); - return [config, &context](Envoy::Network::FilterManager& filter_manager) -> void { + + std::shared_ptr route_config_provider_manager = + context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(generic_route_config_provider_manager), + [&context] { return std::make_shared(context.admin()); }); + + const auto config = + std::make_shared(proto_config, context, *route_config_provider_manager); + return [route_config_provider_manager, config, + &context](Envoy::Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared(config, context)); }; } diff --git a/contrib/generic_proxy/filters/network/source/interface/BUILD b/contrib/generic_proxy/filters/network/source/interface/BUILD index fd60ca0ad124..f04af0b80806 100644 --- a/contrib/generic_proxy/filters/network/source/interface/BUILD +++ b/contrib/generic_proxy/filters/network/source/interface/BUILD @@ -50,6 +50,7 @@ envoy_cc_library( "//envoy/config:typed_metadata_interface", "//envoy/event:dispatcher_interface", "//envoy/network:connection_interface", + "//envoy/rds:rds_config_interface", "//envoy/stream_info:stream_info_interface", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/contrib/generic_proxy/filters/network/source/interface/route.h b/contrib/generic_proxy/filters/network/source/interface/route.h index bc4aa9e1edc4..688d9612f5ed 100644 --- a/contrib/generic_proxy/filters/network/source/interface/route.h +++ b/contrib/generic_proxy/filters/network/source/interface/route.h @@ -4,6 +4,7 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/typed_metadata.h" +#include "envoy/rds/config.h" #include "contrib/generic_proxy/filters/network/source/interface/stream.h" @@ -45,10 +46,8 @@ class RouteEntry { }; using RouteEntryConstSharedPtr = std::shared_ptr; -class RouteMatcher { +class RouteMatcher : public Rds::Config { public: - virtual ~RouteMatcher() = default; - virtual RouteEntryConstSharedPtr routeEntry(const Request& request) const PURE; }; using RouteMatcherPtr = std::unique_ptr; diff --git a/contrib/generic_proxy/filters/network/source/proxy.cc b/contrib/generic_proxy/filters/network/source/proxy.cc index a2cf09d5b5b1..3b06ad8be541 100644 --- a/contrib/generic_proxy/filters/network/source/proxy.cc +++ b/contrib/generic_proxy/filters/network/source/proxy.cc @@ -27,10 +27,26 @@ CodecFactoryPtr FilterConfig::codecFactoryFromProto( return factory.createFactory(*message, context); } -RouteMatcherPtr -FilterConfig::routeMatcherFromProto(const RouteConfiguration& route_config, - Envoy::Server::Configuration::FactoryContext& context) { - return std::make_unique(route_config, context); +Rds::RouteConfigProviderSharedPtr FilterConfig::routeConfigProviderFromProto( + const ProxyConfig& config, Server::Configuration::FactoryContext& context, + RouteConfigProviderManager& route_config_provider_manager) { + if (config.has_generic_rds()) { + if (config.generic_rds().config_source().config_source_specifier_case() == + envoy::config::core::v3::ConfigSource::kApiConfigSource) { + const auto api_type = config.generic_rds().config_source().api_config_source().api_type(); + if (api_type != envoy::config::core::v3::ApiConfigSource::AGGREGATED_GRPC && + api_type != envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC) { + throw EnvoyException("genericrds supports only aggregated api_type in api_config_source"); + } + } + + return route_config_provider_manager.createRdsRouteConfigProvider( + config.generic_rds(), context.getServerFactoryContext(), config.stat_prefix(), + context.initManager()); + } else { + return route_config_provider_manager.createStaticRouteConfigProvider( + config.route_config(), context.getServerFactoryContext()); + } } std::vector FilterConfig::filtersFactoryFromProto( @@ -113,7 +129,7 @@ void ActiveStream::continueDecoding() { } if (cached_route_entry_ == nullptr) { - cached_route_entry_ = parent_.config_->route_matcher_->routeEntry(*downstream_request_stream_); + cached_route_entry_ = parent_.config_->routeEntry(*downstream_request_stream_); } ASSERT(downstream_request_stream_ != nullptr); diff --git a/contrib/generic_proxy/filters/network/source/proxy.h b/contrib/generic_proxy/filters/network/source/proxy.h index f054584b08a6..036f623f33db 100644 --- a/contrib/generic_proxy/filters/network/source/proxy.h +++ b/contrib/generic_proxy/filters/network/source/proxy.h @@ -20,6 +20,8 @@ #include "contrib/generic_proxy/filters/network/source/interface/filter.h" #include "contrib/generic_proxy/filters/network/source/interface/route.h" #include "contrib/generic_proxy/filters/network/source/interface/stream.h" +#include "contrib/generic_proxy/filters/network/source/rds.h" +#include "contrib/generic_proxy/filters/network/source/rds_impl.h" #include "contrib/generic_proxy/filters/network/source/route.h" namespace Envoy { @@ -41,19 +43,22 @@ struct NamedFilterFactoryCb { class FilterConfig : public FilterChainFactory { public: - FilterConfig(const std::string& stat_prefix, CodecFactoryPtr codec, RouteMatcherPtr route_matcher, - std::vector factories, Server::Configuration::FactoryContext&) + FilterConfig(const std::string& stat_prefix, CodecFactoryPtr codec, + Rds::RouteConfigProviderSharedPtr route_config_provider, + std::vector factories) : stat_prefix_(stat_prefix), codec_factory_(std::move(codec)), - route_matcher_(std::move(route_matcher)), factories_(std::move(factories)) {} + route_config_provider_(std::move(route_config_provider)), factories_(std::move(factories)) { + } - FilterConfig(const ProxyConfig& config, Server::Configuration::FactoryContext& context) + FilterConfig(const ProxyConfig& config, Server::Configuration::FactoryContext& context, + RouteConfigProviderManager& route_config_provider_manager) : FilterConfig(config.stat_prefix(), codecFactoryFromProto(config.codec_config(), context), - routeMatcherFromProto(config.route_config(), context), - filtersFactoryFromProto(config.filters(), config.stat_prefix(), context), - context) {} + routeConfigProviderFromProto(config, context, route_config_provider_manager), + filtersFactoryFromProto(config.filters(), config.stat_prefix(), context)) {} RouteEntryConstSharedPtr routeEntry(const Request& request) const { - return route_matcher_->routeEntry(request); + auto config = std::static_pointer_cast(route_config_provider_->config()); + return config->routeEntry(request); } // FilterChainFactory @@ -69,8 +74,10 @@ class FilterConfig : public FilterChainFactory { codecFactoryFromProto(const envoy::config::core::v3::TypedExtensionConfig& codec_config, Server::Configuration::FactoryContext& context); - static RouteMatcherPtr routeMatcherFromProto(const RouteConfiguration& route_config, - Server::Configuration::FactoryContext& context); + static Rds::RouteConfigProviderSharedPtr + routeConfigProviderFromProto(const ProxyConfig& config, + Server::Configuration::FactoryContext& context, + RouteConfigProviderManager& route_config_provider_manager); static std::vector filtersFactoryFromProto( const ProtobufWkt::RepeatedPtrField& filters, @@ -84,7 +91,7 @@ class FilterConfig : public FilterChainFactory { CodecFactoryPtr codec_factory_; - RouteMatcherPtr route_matcher_; + Rds::RouteConfigProviderSharedPtr route_config_provider_; std::vector factories_; }; diff --git a/contrib/generic_proxy/filters/network/source/rds.h b/contrib/generic_proxy/filters/network/source/rds.h new file mode 100644 index 000000000000..301239728989 --- /dev/null +++ b/contrib/generic_proxy/filters/network/source/rds.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include "source/common/rds/common/route_config_provider_manager.h" + +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.pb.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.pb.validate.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/route.pb.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/route.pb.validate.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace GenericProxy { + +using RouteConfigProviderManager = Rds::Common::RouteConfigProviderManager< + envoy::extensions::filters::network::generic_proxy::v3::GenericRds, + envoy::extensions::filters::network::generic_proxy::v3::RouteConfiguration>; + +} // namespace GenericProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/generic_proxy/filters/network/source/rds_impl.h b/contrib/generic_proxy/filters/network/source/rds_impl.h new file mode 100644 index 000000000000..39248e9c2343 --- /dev/null +++ b/contrib/generic_proxy/filters/network/source/rds_impl.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include "source/common/rds/common/route_config_provider_manager_impl.h" + +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.pb.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.pb.validate.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/route.pb.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/route.pb.validate.h" +#include "contrib/generic_proxy/filters/network/source/route.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace GenericProxy { + +using RouteConfigProviderManagerImpl = Rds::Common::RouteConfigProviderManagerImpl< + envoy::extensions::filters::network::generic_proxy::v3::GenericRds, + envoy::extensions::filters::network::generic_proxy::v3::RouteConfiguration, 1, RouteMatcherImpl, + NullRouteMatcherImpl>; + +} // namespace GenericProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/generic_proxy/filters/network/source/route.cc b/contrib/generic_proxy/filters/network/source/route.cc index 0983e1c36730..d2eb2df5aae6 100644 --- a/contrib/generic_proxy/filters/network/source/route.cc +++ b/contrib/generic_proxy/filters/network/source/route.cc @@ -50,14 +50,15 @@ Matcher::ActionFactoryCb RouteMatchActionFactory::createActionFactoryCb( REGISTER_FACTORY(RouteMatchActionFactory, Matcher::ActionFactory); RouteMatcherImpl::RouteMatcherImpl(const ProtoRouteConfiguration& route_config, - Envoy::Server::Configuration::FactoryContext& context) + Envoy::Server::Configuration::ServerFactoryContext& context, + bool) : name_(route_config.name()) { RouteActionValidationVisitor validation_visitor; - RouteActionContext action_context{context.getServerFactoryContext()}; + RouteActionContext action_context{context}; - Matcher::MatchTreeFactory factory( - action_context, context.getServerFactoryContext(), validation_visitor); + Matcher::MatchTreeFactory factory(action_context, context, + validation_visitor); matcher_ = factory.create(route_config.routes())(); diff --git a/contrib/generic_proxy/filters/network/source/route.h b/contrib/generic_proxy/filters/network/source/route.h index 096d810998d5..a4ca169bcd93 100644 --- a/contrib/generic_proxy/filters/network/source/route.h +++ b/contrib/generic_proxy/filters/network/source/route.h @@ -88,10 +88,17 @@ class RouteMatchActionFactory : public Matcher::ActionFactory { public: RouteMatcherImpl(const ProtoRouteConfiguration& route_config, - Envoy::Server::Configuration::FactoryContext& context); + Envoy::Server::Configuration::ServerFactoryContext& context, + bool validate_clusters_default = false); RouteEntryConstSharedPtr routeEntry(const Request& request) const override; diff --git a/contrib/generic_proxy/filters/network/test/BUILD b/contrib/generic_proxy/filters/network/test/BUILD index 63aff37747d0..7478e5b5034b 100644 --- a/contrib/generic_proxy/filters/network/test/BUILD +++ b/contrib/generic_proxy/filters/network/test/BUILD @@ -68,6 +68,8 @@ envoy_cc_test( "//test/mocks/server:factory_context_mocks", "//test/test_common:registry_lib", "//test/test_common:utility_lib", + "@envoy_api//contrib/envoy/extensions/filters/network/generic_proxy/v3:pkg_cc_proto", + "@envoy_api//envoy/admin/v3:pkg_cc_proto", ], ) diff --git a/contrib/generic_proxy/filters/network/test/config_test.cc b/contrib/generic_proxy/filters/network/test/config_test.cc index dfd8027291ac..e9d05ef29b8f 100644 --- a/contrib/generic_proxy/filters/network/test/config_test.cc +++ b/contrib/generic_proxy/filters/network/test/config_test.cc @@ -1,7 +1,14 @@ +#include "envoy/admin/v3/config_dump_shared.pb.h" +#include "envoy/admin/v3/config_dump_shared.pb.validate.h" + #include "test/mocks/server/factory_context.h" #include "test/test_common/registry.h" #include "test/test_common/utility.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.pb.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.pb.validate.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/route.pb.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/v3/route.pb.validate.h" #include "contrib/generic_proxy/filters/network/source/config.h" #include "contrib/generic_proxy/filters/network/test/fake_codec.h" #include "gtest/gtest.h" @@ -69,6 +76,94 @@ TEST(FactoryTest, FactoryTest) { EXPECT_NE(nullptr, factory.createFilterFactoryFromProto(proto_config, factory_context)); } +TEST(FactoryTest, GenericRds) { + const std::string config_yaml = R"EOF( + stat_prefix: ingress + filters: + - name: envoy.filters.generic.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.router.v3.Router + codec_config: + name: fake + typed_config: + "@type": type.googleapis.com/xds.type.v3.TypedStruct + type_url: envoy.generic_proxy.codecs.fake.type + value: {} + generic_rds: + config_source: { resource_api_version: V3, ads: {} } + route_config_name: test_route + )EOF"; + + const std::string response_yaml = (R"EOF( +version_info: "1" +resources: + - "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.v3.RouteConfiguration + name: test_route + routes: {} +)EOF"); + + FakeStreamCodecFactoryConfig codec_factory_config; + Registry::InjectFactory registration(codec_factory_config); + + NiceMock factory_context; + + Factory factory; + + envoy::extensions::filters::network::generic_proxy::v3::GenericProxy config; + TestUtility::loadFromYaml(config_yaml, config); + + Matchers::UniversalStringMatcher universal_name_matcher; + Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, factory_context); + auto response = + TestUtility::parseYaml(response_yaml); + const auto decoded_resources = TestUtility::decodeResources< + envoy::extensions::filters::network::generic_proxy::v3::RouteConfiguration>(response); + factory_context.server_factory_context_.cluster_manager_.subscription_factory_.callbacks_ + ->onConfigUpdate(decoded_resources.refvec_, response.version_info()); + auto message_ptr = + factory_context.admin_.config_tracker_.config_tracker_callbacks_["genericrds_routes"]( + universal_name_matcher); + const auto& dump = + TestUtility::downcastAndValidate(*message_ptr); + EXPECT_EQ(1, dump.dynamic_route_configs().size()); + EXPECT_EQ(0, dump.static_route_configs().size()); +} + +TEST(FactoryTest, GenericRdsApiConfigSource) { + const std::string config_yaml = R"EOF( + stat_prefix: ingress + filters: + - name: envoy.filters.generic.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.router.v3.Router + codec_config: + name: fake + typed_config: + "@type": type.googleapis.com/xds.type.v3.TypedStruct + type_url: envoy.generic_proxy.codecs.fake.type + value: {} + generic_rds: + config_source: + resource_api_version: V3 + api_config_source: { api_type: GRPC, transport_api_version: V3 } + route_config_name: test_route + )EOF"; + + FakeStreamCodecFactoryConfig codec_factory_config; + Registry::InjectFactory registration(codec_factory_config); + + NiceMock factory_context; + + Factory factory; + + envoy::extensions::filters::network::generic_proxy::v3::GenericProxy config; + TestUtility::loadFromYaml(config_yaml, config); + + EXPECT_THROW_WITH_REGEX(factory.createFilterFactoryFromProto(config, factory_context), + EnvoyException, + "genericrds supports only aggregated api_type in api_config_source"); +} + } // namespace } // namespace GenericProxy } // namespace NetworkFilters diff --git a/contrib/generic_proxy/filters/network/test/proxy_test.cc b/contrib/generic_proxy/filters/network/test/proxy_test.cc index aedf700c8c89..ea156b33cd1c 100644 --- a/contrib/generic_proxy/filters/network/test/proxy_test.cc +++ b/contrib/generic_proxy/filters/network/test/proxy_test.cc @@ -11,6 +11,7 @@ #include "contrib/generic_proxy/filters/network/test/mocks/codec.h" #include "contrib/generic_proxy/filters/network/test/mocks/filter.h" #include "contrib/generic_proxy/filters/network/test/mocks/route.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" using testing::ByMove; @@ -23,6 +24,18 @@ namespace NetworkFilters { namespace GenericProxy { namespace { +class MockRouteConfigProvider : public Rds::RouteConfigProvider { +public: + MockRouteConfigProvider() { ON_CALL(*this, config()).WillByDefault(Return(route_config_)); } + + MOCK_METHOD(Rds::ConfigConstSharedPtr, config, (), (const)); + MOCK_METHOD(const absl::optional&, configInfo, (), (const)); + MOCK_METHOD(SystemTime, lastUpdated, (), (const)); + MOCK_METHOD(void, onConfigUpdate, ()); + + std::shared_ptr> route_config_{new NiceMock()}; +}; + /** * Test creating codec factory from typed extension config. */ @@ -65,44 +78,6 @@ TEST(BasicFilterConfigTest, CreatingCodecFactory) { } } -/** - * Test creating route matcher from proto config. - */ -TEST(BasicFilterConfigTest, CreatingRouteMatcher) { - static const std::string yaml_config = R"EOF( - name: test_matcher_tree - routes: - matcher_list: - matchers: - - predicate: - and_matcher: - predicate: - - single_predicate: - input: - name: envoy.matching.generic_proxy.input.service - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.ServiceMatchInput - value_match: - exact: "service_0" - on_match: - action: - name: envoy.matching.action.generic_proxy.route - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction - cluster: "cluster_0" - metadata: - filter_metadata: - mock_filter: - key_0: value_0 - )EOF"; - NiceMock factory_context; - - ProtoRouteConfiguration proto_config; - TestUtility::loadFromYaml(yaml_config, proto_config); - - EXPECT_NE(nullptr, FilterConfig::routeMatcherFromProto(proto_config, factory_context)); -} - /** * Test creating L7 filter factories from proto config. */ @@ -195,21 +170,20 @@ class FilterConfigTest : public testing::Test { auto codec_factory = std::make_unique>(); codec_factory_ = codec_factory.get(); - auto route_matcher = std::make_unique>(); - route_matcher_ = route_matcher.get(); - mock_route_entry_ = std::make_shared>(); - filter_config_ = - std::make_shared("test_prefix", std::move(codec_factory), - std::move(route_matcher), factories, factory_context_); + filter_config_ = std::make_shared("test_prefix", std::move(codec_factory), + route_config_provider_, factories); } std::shared_ptr filter_config_; NiceMock factory_context_; + std::shared_ptr> route_config_provider_{ + new NiceMock()}; + NiceMock* route_matcher_ = route_config_provider_->route_config_.get(); + NiceMock* codec_factory_; - NiceMock* route_matcher_; using MockStreamFilterSharedPtr = std::shared_ptr>; using MockDecoderFilterSharedPtr = std::shared_ptr>; diff --git a/contrib/generic_proxy/filters/network/test/route_test.cc b/contrib/generic_proxy/filters/network/test/route_test.cc index 14be4ad3d36a..416b2abcdc29 100644 --- a/contrib/generic_proxy/filters/network/test/route_test.cc +++ b/contrib/generic_proxy/filters/network/test/route_test.cc @@ -211,7 +211,7 @@ class RouteMatcherImplTest : public testing::Test { } protected: - NiceMock factory_context_; + NiceMock factory_context_; std::unique_ptr route_matcher_; }; From 60e0a125d84488e5aaf1e60d1770a672076c5e11 Mon Sep 17 00:00:00 2001 From: David Schinazi Date: Wed, 2 Nov 2022 18:42:23 -0700 Subject: [PATCH 091/112] Update QUICHE from 9624f2d43 to c6efbc4f7 (#23810) https://github.com/google/quiche/compare/9624f2d43..c6efbc4f7 $ git log 9624f2d43..c6efbc4f7 --date=short --no-merges --format="%ad %al %s" 2022-11-02 quiche-dev Add QUICHE_EXPORT to quiche::ObliviousHttpRequest::Context 2022-11-02 ramosalex Decouple bind interface from socket creation to more easily reuse in different implementations. 2022-11-02 dschinazi Fixup logs 2022-11-02 dschinazi CONNECT-IP server support 2022-11-02 fayang Delay setting stateless reset token in config until session initialization. This avoids virtual function GetStatelessResetToken be called in QuicSession constructor. 2022-11-01 quiche-dev Add QUICHE_EXPORT to the public inner class in binary_http_message.h 2022-11-01 wub Add more debug information in QuicStreamSequencer::DebugString: - First missing byte - Next expected byte - Received frames 2022-10-31 quiche-dev Internal change 2022-10-31 birenroy Initializes the memory backing CallbackVisitor::current_frame_ in the constructor. 2022-10-29 dschinazi CONNECT-IP encapsulated client support 2022-10-28 quiche-dev Take absl::string_view by value instead of const-ref in ParseDictionary 2022-10-28 bschneider Reduce string copies in Ohttp response logic. 2022-10-28 birenroy Rejects WINDOW_UPDATE frames that would cause a window overflow, as per Section 6.9.1 of RFC 9113. 2022-10-28 birenroy Rejects WINDOW_UPDATE frames with zero deltas, as per RFC 9113 Section 6.9. 2022-10-27 dschinazi Properly support RFC 9297 Signed-off-by: David Schinazi Signed-off-by: David Schinazi --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 86f4c1c7b37b..93f9763fda83 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1080,12 +1080,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "9624f2d439f5531ef92346f18d337270cca4ee23", - sha256 = "6d30f11ee198751851b83c50448e9fe9d62dc55c9f897f40be9b35a6fc49039b", + version = "c6efbc4f790a274f1f4030cd8437683a321f23b8", + sha256 = "7911438519437af82356c76a9b96a8a61fcb3ee5c11ca7e6f190e4bca53c3cb0", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["dataplane_core"], - release_date = "2022-10-27", + release_date = "2022-11-02", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", From 860b750b8c6fec57bdf12c58f872defbc56d4f73 Mon Sep 17 00:00:00 2001 From: code Date: Thu, 3 Nov 2022 17:28:20 +0800 Subject: [PATCH 092/112] tracing: update the tracingConfig() to return opt ref (#23633) * tracing: update the tracingConfig() to return opt ref Signed-off-by: wbpcode * minor updates to address review comments Signed-off-by: wbpcode * add test to ensure the child span of active span will be null span when the hcm tracing config is null Signed-off-by: wbpcode Signed-off-by: wbpcode --- envoy/http/filter.h | 4 +- source/common/http/async_client_impl.h | 4 +- source/common/http/conn_manager_impl.cc | 46 ++++++++++--------- source/common/http/conn_manager_impl.h | 19 ++++---- source/common/http/filter_manager.cc | 2 +- source/common/http/filter_manager.h | 4 +- source/common/router/upstream_request.cc | 22 +++++---- source/common/router/upstream_request.h | 2 +- test/common/http/conn_manager_impl_test.cc | 51 +++++++++++++++++++++ test/common/router/upstream_request_test.cc | 4 +- test/mocks/http/mocks.cc | 6 ++- test/mocks/http/mocks.h | 6 +-- 12 files changed, 117 insertions(+), 53 deletions(-) diff --git a/envoy/http/filter.h b/envoy/http/filter.h index 606cc8466bed..10ba0ec44c69 100644 --- a/envoy/http/filter.h +++ b/envoy/http/filter.h @@ -358,9 +358,9 @@ class StreamFilterCallbacks { virtual Tracing::Span& activeSpan() PURE; /** - * @return tracing configuration. + * @return tracing configuration if present. */ - virtual const Tracing::Config& tracingConfig() PURE; + virtual OptRef tracingConfig() const PURE; /** * @return the ScopeTrackedObject for this stream. diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index d848b5dceafe..00f6edf027a5 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -363,7 +363,9 @@ class AsyncStreamImpl : public AsyncClient::Stream, // TODO(kbaichoo): Plumb account from owning request filter. Buffer::BufferMemoryAccountSharedPtr account() const override { return nullptr; } Tracing::Span& activeSpan() override { return active_span_; } - const Tracing::Config& tracingConfig() override { return tracing_config_; } + OptRef tracingConfig() const override { + return makeOptRef(tracing_config_); + } void continueDecoding() override {} RequestTrailerMap& addDecodedTrailers() override { PANIC("not implemented"); } void addDecodedData(Buffer::Instance&, bool) override { diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 565685ef26b2..29917f683bc8 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -648,6 +648,10 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect uint32_t buffer_limit, Buffer::BufferMemoryAccountSharedPtr account) : connection_manager_(connection_manager), + connection_manager_tracing_config_(connection_manager_.config_.tracingConfig() == nullptr + ? absl::nullopt + : makeOptRef( + *connection_manager_.config_.tracingConfig())), stream_id_(connection_manager.random_generator_.random()), filter_manager_(*this, connection_manager_.read_callbacks_->connection().dispatcher(), connection_manager_.read_callbacks_->connection(), stream_id_, @@ -1103,8 +1107,8 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapPtr&& he // Allow non websocket requests to go through websocket enabled routes. } - // Check if tracing is enabled at all. - if (connection_manager_.config_.tracingConfig()) { + // Check if tracing is enabled. + if (connection_manager_tracing_config_.has_value()) { traceRequest(); } @@ -1144,8 +1148,7 @@ void ConnectionManagerImpl::ActiveStream::traceRequest() { } } - if (connection_manager_.config_.tracingConfig()->operation_name_ == - Tracing::OperationName::Egress) { + if (connection_manager_tracing_config_->operation_name_ == Tracing::OperationName::Egress) { // For egress (outbound) requests, pass the decorator's operation name (if defined and // propagation enabled) as a request header to enable the receiving service to use it in its // server span. @@ -1334,11 +1337,10 @@ void ConnectionManagerImpl::ActiveStream::refreshCachedRoute(const Router::Route } void ConnectionManagerImpl::ActiveStream::refreshCachedTracingCustomTags() { - if (!connection_manager_.config_.tracingConfig()) { + if (!connection_manager_tracing_config_.has_value()) { return; } - const Tracing::CustomTagMap& conn_manager_tags = - connection_manager_.config_.tracingConfig()->custom_tags_; + const Tracing::CustomTagMap& conn_manager_tags = connection_manager_tracing_config_->custom_tags_; const Tracing::CustomTagMap* route_tags = nullptr; if (hasCachedRoute() && cached_route_.value()->tracingConfig()) { route_tags = &cached_route_.value()->tracingConfig()->getCustomTags(); @@ -1484,9 +1486,8 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ResponseHeaderMap& heade } } - if (connection_manager_.config_.tracingConfig()) { - if (connection_manager_.config_.tracingConfig()->operation_name_ == - Tracing::OperationName::Ingress) { + if (connection_manager_tracing_config_.has_value()) { + if (connection_manager_tracing_config_->operation_name_ == Tracing::OperationName::Ingress) { // For ingress (inbound) responses, if the request headers do not include a // decorator operation (override), and the decorated operation should be // propagated, then pass the decorator's operation name (if defined) @@ -1494,7 +1495,7 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ResponseHeaderMap& heade if (decorated_operation_ && state_.decorated_propagate_) { headers.setEnvoyDecoratorOperation(*decorated_operation_); } - } else if (connection_manager_.config_.tracingConfig()->operation_name_ == + } else if (connection_manager_tracing_config_->operation_name_ == Tracing::OperationName::Egress) { const HeaderEntry* resp_operation_override = headers.EnvoyDecoratorOperation(); @@ -1601,10 +1602,8 @@ void ConnectionManagerImpl::ActiveStream::onBelowWriteBufferLowWatermark() { } Tracing::OperationName ConnectionManagerImpl::ActiveStream::operationName() const { - if (!connection_manager_.config_.tracingConfig()) { - return Tracing::OperationName::Egress; - } - return connection_manager_.config_.tracingConfig()->operation_name_; + ASSERT(connection_manager_tracing_config_.has_value()); + return connection_manager_tracing_config_->operation_name_; } const Tracing::CustomTagMap* ConnectionManagerImpl::ActiveStream::customTags() const { @@ -1612,15 +1611,13 @@ const Tracing::CustomTagMap* ConnectionManagerImpl::ActiveStream::customTags() c } bool ConnectionManagerImpl::ActiveStream::verbose() const { - return connection_manager_.config_.tracingConfig() && - connection_manager_.config_.tracingConfig()->verbose_; + ASSERT(connection_manager_tracing_config_.has_value()); + return connection_manager_tracing_config_->verbose_; } uint32_t ConnectionManagerImpl::ActiveStream::maxPathTagLength() const { - if (!connection_manager_.config_.tracingConfig()) { - return Tracing::DefaultMaxPathTagLength; - } - return connection_manager_.config_.tracingConfig()->max_path_tag_length_; + ASSERT(connection_manager_tracing_config_.has_value()); + return connection_manager_tracing_config_->max_path_tag_length_; } const Router::RouteEntry::UpgradeMap* ConnectionManagerImpl::ActiveStream::upgradeMap() { @@ -1641,7 +1638,12 @@ Tracing::Span& ConnectionManagerImpl::ActiveStream::activeSpan() { } } -Tracing::Config& ConnectionManagerImpl::ActiveStream::tracingConfig() { return *this; } +OptRef ConnectionManagerImpl::ActiveStream::tracingConfig() const { + if (connection_manager_tracing_config_.has_value()) { + return makeOptRef(*this); + } + return {}; +} const ScopeTrackedObject& ConnectionManagerImpl::ActiveStream::scope() { return *this; } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 583bea311d92..b9c6402a23ea 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -202,12 +202,6 @@ class ConnectionManagerImpl : Logger::Loggable, return filter_manager_.sendLocalReply(code, body, modify_headers, grpc_status, details); } - // Tracing::TracingConfig - Tracing::OperationName operationName() const override; - const Tracing::CustomTagMap* customTags() const override; - bool verbose() const override; - uint32_t maxPathTagLength() const override; - // ScopeTrackedObject void dumpState(std::ostream& os, int indent_level = 0) const override { const char* spaces = spacesForLevel(indent_level); @@ -283,8 +277,7 @@ class ConnectionManagerImpl : Logger::Loggable, void onRequestDataTooLarge() override; Http1StreamEncoderOptionsOptRef http1StreamEncoderOptions() override; void onLocalReply(Code code) override; - // TODO(alyssawilk) this should be an optional reference. - Tracing::Config& tracingConfig() override; + OptRef tracingConfig() const override; const ScopeTrackedObject& scope() override; OptRef downstreamCallbacks() override { return *this; } @@ -363,6 +356,7 @@ class ConnectionManagerImpl : Logger::Loggable, bool validateHeaders(); ConnectionManagerImpl& connection_manager_; + OptRef connection_manager_tracing_config_; // TODO(snowp): It might make sense to move this to the FilterManager to avoid storing it in // both locations, then refer to the FM when doing stream logs. const uint64_t stream_id_; @@ -406,6 +400,15 @@ class ConnectionManagerImpl : Logger::Loggable, Http::HeaderValidatorPtr header_validator_; friend FilterManager; + + private: + // Keep these methods private to ensure that these methods are only called by the reference + // returned by the public tracingConfig() method. + // Tracing::TracingConfig + Tracing::OperationName operationName() const override; + const Tracing::CustomTagMap* customTags() const override; + bool verbose() const override; + uint32_t maxPathTagLength() const override; }; using ActiveStreamPtr = std::unique_ptr; diff --git a/source/common/http/filter_manager.cc b/source/common/http/filter_manager.cc index a1ac6acea529..c3a8b3bd1008 100644 --- a/source/common/http/filter_manager.cc +++ b/source/common/http/filter_manager.cc @@ -253,7 +253,7 @@ void ActiveStreamFilterBase::restoreContextOnContinue( parent_.contextOnContinue(tracked_object_stack); } -const Tracing::Config& ActiveStreamFilterBase::tracingConfig() { +OptRef ActiveStreamFilterBase::tracingConfig() const { return parent_.filter_manager_callbacks_.tracingConfig(); } diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h index d4c13ff561dd..1bedb7ff90a3 100644 --- a/source/common/http/filter_manager.h +++ b/source/common/http/filter_manager.h @@ -92,7 +92,7 @@ struct ActiveStreamFilterBase : public virtual StreamFilterCallbacks, uint64_t streamId() const override; StreamInfo::StreamInfo& streamInfo() override; Tracing::Span& activeSpan() override; - const Tracing::Config& tracingConfig() override; + OptRef tracingConfig() const override; const ScopeTrackedObject& scope() override; void restoreContextOnContinue(ScopeTrackedObjectStack& tracked_object_stack) override; void resetIdleTimer() override; @@ -504,7 +504,7 @@ class FilterManagerCallbacks { /** * Returns the tracing configuration to use for this stream. */ - virtual const Tracing::Config& tracingConfig() PURE; + virtual OptRef tracingConfig() const PURE; /** * Returns the tracked scope to use for this stream. diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index 178e06a29f59..5407c2ce1d28 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -94,13 +94,15 @@ UpstreamRequest::UpstreamRequest(RouterFilterInterface& parent, Runtime::runtimeFeatureEnabled("envoy.reloadable_features.allow_upstream_filters")), stream_options_({can_send_early_data, can_use_http3}) { if (parent_.config().start_child_span_) { - span_ = parent_.callbacks()->activeSpan().spawnChild( - parent_.callbacks()->tracingConfig(), - absl::StrCat("router ", parent.cluster()->observabilityName(), " egress"), - parent.timeSource().systemTime()); - if (parent.attemptCount() != 1) { - // This is a retry request, add this metadata to span. - span_->setTag(Tracing::Tags::get().RetryCount, std::to_string(parent.attemptCount() - 1)); + if (auto tracing_config = parent_.callbacks()->tracingConfig(); tracing_config.has_value()) { + span_ = parent_.callbacks()->activeSpan().spawnChild( + tracing_config.value().get(), + absl::StrCat("router ", parent.cluster()->observabilityName(), " egress"), + parent.timeSource().systemTime()); + if (parent.attemptCount() != 1) { + // This is a retry request, add this metadata to span. + span_->setTag(Tracing::Tags::get().RetryCount, std::to_string(parent.attemptCount() - 1)); + } } } stream_info_.setUpstreamInfo(std::make_shared()); @@ -142,8 +144,10 @@ void UpstreamRequest::cleanUp() { } if (span_ != nullptr) { + auto tracing_config = parent_.callbacks()->tracingConfig(); + ASSERT(tracing_config.has_value()); Tracing::HttpTracerUtility::finalizeUpstreamSpan(*span_, stream_info_, - parent_.callbacks()->tracingConfig()); + tracing_config.value().get()); } if (per_try_timeout_ != nullptr) { @@ -842,7 +846,7 @@ const ScopeTrackedObject& UpstreamRequestFilterManagerCallbacks::scope() { return upstream_request_.parent_.callbacks()->scope(); } -const Tracing::Config& UpstreamRequestFilterManagerCallbacks::tracingConfig() { +OptRef UpstreamRequestFilterManagerCallbacks::tracingConfig() const { return upstream_request_.parent_.callbacks()->tracingConfig(); } diff --git a/source/common/router/upstream_request.h b/source/common/router/upstream_request.h index 3411a5019154..0a119321d7da 100644 --- a/source/common/router/upstream_request.h +++ b/source/common/router/upstream_request.h @@ -322,7 +322,7 @@ class UpstreamRequestFilterManagerCallbacks : public Http::FilterManagerCallback } // These functions are delegated to the downstream HCM/FM - const Tracing::Config& tracingConfig() override; + OptRef tracingConfig() const override; const ScopeTrackedObject& scope() override; Tracing::Span& activeSpan() override; void resetStream(Http::StreamResetReason reset_reason, diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index e1ba272cc749..32d31f151d8b 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -1942,6 +1942,57 @@ TEST_F(HttpConnectionManagerImplTest, conn_manager_->onData(fake_input, false); } +TEST_F(HttpConnectionManagerImplTest, NoHCMTracingConfigAndActiveSpanWouldBeNullSpan) { + setup(false, ""); + // Null HCM tracing config. + tracing_config_ = nullptr; + + std::shared_ptr filter(new NiceMock()); + + EXPECT_CALL(filter_factory_, createFilterChain(_)) + .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> void { + auto factory = createDecoderFilterFactoryCb(filter); + manager.applyFilterFactoryCb({}, factory); + })); + + // Treat request as internal, otherwise x-request-id header will be overwritten. + use_remote_address_ = false; + EXPECT_CALL(random_, uuid()).Times(0); + + EXPECT_CALL(*codec_, dispatch(_)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data) -> Http::Status { + decoder_ = &conn_manager_->newStream(response_encoder_); + + RequestHeaderMapPtr headers{ + new TestRequestHeaderMapImpl{{":method", "GET"}, + {":authority", "host"}, + {":path", "/"}, + {"x-request-id", "125a4afb-6f55-a4ba-ad80-413f09f48a28"}}}; + decoder_->decodeHeaders(std::move(headers), true); + + filter->callbacks_->streamInfo().setResponseCodeDetails(""); + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{ + {":status", "200"}, {"x-envoy-decorator-operation", "testOp"}}}; + filter->callbacks_->encodeHeaders(std::move(response_headers), true, "details"); + + // Active span always be null span when there is no HCM tracing config. + EXPECT_EQ(&Tracing::NullSpan::instance(), &filter->callbacks_->activeSpan()); + // Child span should also be null span. + Tracing::SpanPtr child_span = filter->callbacks_->activeSpan().spawnChild( + Tracing::EgressConfig::get(), "null_child", test_time_.systemTime()); + Tracing::SpanPtr null_span = std::make_unique(); + auto& child_span_ref = *child_span; + auto& null_span_ref = *null_span; + EXPECT_EQ(typeid(child_span_ref).name(), typeid(null_span_ref).name()); + + data.drain(4); + return Http::okStatus(); + })); + + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); +} + TEST_F(HttpConnectionManagerImplTest, TestAccessLog) { static constexpr char remote_address[] = "0.0.0.0"; static constexpr char xff_address[] = "1.2.3.4"; diff --git a/test/common/router/upstream_request_test.cc b/test/common/router/upstream_request_test.cc index 02380b89ffe2..88ab1bff7e3d 100644 --- a/test/common/router/upstream_request_test.cc +++ b/test/common/router/upstream_request_test.cc @@ -140,8 +140,8 @@ TEST_F(UpstreamRequestTest, AcceptRouterHeaders) { EXPECT_FALSE(filter->callbacks_->informationalHeaders().has_value()); EXPECT_FALSE(filter->callbacks_->responseHeaders().has_value()); EXPECT_FALSE(filter->callbacks_->http1StreamEncoderOptions().has_value()); - EXPECT_EQ(&filter->callbacks_->tracingConfig(), - &router_filter_interface_.callbacks_.tracingConfig()); + EXPECT_EQ(&filter->callbacks_->tracingConfig().value().get(), + &router_filter_interface_.callbacks_.tracingConfig().value().get()); EXPECT_EQ(filter->callbacks_->clusterInfo(), router_filter_interface_.callbacks_.clusterInfo()); EXPECT_EQ(&filter->callbacks_->activeSpan(), &router_filter_interface_.callbacks_.activeSpan()); EXPECT_EQ(&filter->callbacks_->streamInfo(), &router_filter_interface_.callbacks_.streamInfo()); diff --git a/test/mocks/http/mocks.cc b/test/mocks/http/mocks.cc index 89953c910f2a..318af3f218f4 100644 --- a/test/mocks/http/mocks.cc +++ b/test/mocks/http/mocks.cc @@ -85,7 +85,8 @@ MockStreamDecoderFilterCallbacks::MockStreamDecoderFilterCallbacks() { })); ON_CALL(*this, activeSpan()).WillByDefault(ReturnRef(active_span_)); - ON_CALL(*this, tracingConfig()).WillByDefault(ReturnRef(tracing_config_)); + ON_CALL(*this, tracingConfig()) + .WillByDefault(Return(makeOptRef(tracing_config_))); ON_CALL(*this, scope()).WillByDefault(ReturnRef(scope_)); ON_CALL(*this, sendLocalReply(_, _, _, _, _)) .WillByDefault(Invoke([this](Code code, absl::string_view body, @@ -143,7 +144,8 @@ MockStreamEncoderFilterCallbacks::MockStreamEncoderFilterCallbacks() { initializeMockStreamFilterCallbacks(*this); ON_CALL(*this, encodingBuffer()).WillByDefault(Invoke(&buffer_, &Buffer::InstancePtr::get)); ON_CALL(*this, activeSpan()).WillByDefault(ReturnRef(active_span_)); - ON_CALL(*this, tracingConfig()).WillByDefault(ReturnRef(tracing_config_)); + ON_CALL(*this, tracingConfig()) + .WillByDefault(Return(makeOptRef(tracing_config_))); ON_CALL(*this, scope()).WillByDefault(ReturnRef(scope_)); ON_CALL(*this, mostSpecificPerFilterConfig()) diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 45c8082e4276..9bb2d0a82aad 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -108,7 +108,7 @@ class MockFilterManagerCallbacks : public FilterManagerCallbacks { MOCK_METHOD(OptRef, downstreamCallbacks, ()); MOCK_METHOD(OptRef, upstreamCallbacks, ()); MOCK_METHOD(void, onLocalReply, (Code code)); - MOCK_METHOD(Tracing::Config&, tracingConfig, ()); + MOCK_METHOD(OptRef, tracingConfig, (), (const)); MOCK_METHOD(const ScopeTrackedObject&, scope, ()); MOCK_METHOD(void, restoreContextOnContinue, (ScopeTrackedObjectStack&)); @@ -248,7 +248,7 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, MOCK_METHOD(uint64_t, streamId, (), (const)); MOCK_METHOD(StreamInfo::StreamInfo&, streamInfo, ()); MOCK_METHOD(Tracing::Span&, activeSpan, ()); - MOCK_METHOD(Tracing::Config&, tracingConfig, ()); + MOCK_METHOD(OptRef, tracingConfig, (), (const)); MOCK_METHOD(const ScopeTrackedObject&, scope, ()); MOCK_METHOD(void, restoreContextOnContinue, (ScopeTrackedObjectStack&)); MOCK_METHOD(void, onDecoderFilterAboveWriteBufferHighWatermark, ()); @@ -344,7 +344,7 @@ class MockStreamEncoderFilterCallbacks : public StreamEncoderFilterCallbacks, MOCK_METHOD(uint64_t, streamId, (), (const)); MOCK_METHOD(StreamInfo::StreamInfo&, streamInfo, ()); MOCK_METHOD(Tracing::Span&, activeSpan, ()); - MOCK_METHOD(Tracing::Config&, tracingConfig, ()); + MOCK_METHOD(OptRef, tracingConfig, (), (const)); MOCK_METHOD(const ScopeTrackedObject&, scope, ()); MOCK_METHOD(void, onEncoderFilterAboveWriteBufferHighWatermark, ()); MOCK_METHOD(void, onEncoderFilterBelowWriteBufferLowWatermark, ()); From 6f338ceccb5bcf03fa8b70ba2b02f3e5d174919e Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Thu, 3 Nov 2022 06:45:01 -0700 Subject: [PATCH 093/112] bazel: update googleurl patch with missing header (#23807) Upstream fix: https://quiche-review.googlesource.com/c/googleurl/+/11260 Signed-off-by: Keith Smiley --- bazel/external/googleurl.patch | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/bazel/external/googleurl.patch b/bazel/external/googleurl.patch index 81d35d7d612b..a426657bee04 100644 --- a/bazel/external/googleurl.patch +++ b/bazel/external/googleurl.patch @@ -117,13 +117,21 @@ index f881c04..4e4f7df 100644 // for official builds. // TODO(crbug.com/1320670): Remove when issue is resolved. -# TODO(keith): Remove when https://quiche-review.googlesource.com/c/googleurl/+/11220 lands +# TODO(keith): Remove when https://quiche-review.googlesource.com/c/googleurl/+/11220 and https://quiche-review.googlesource.com/c/googleurl/+/11260 land diff --git a/base/BUILD b/base/BUILD -index 60ca578..6a793a6 100644 +index 60ca578..c922b73 100644 --- a/base/BUILD +++ b/base/BUILD -@@ -55,6 +55,7 @@ cc_library( +@@ -40,6 +40,7 @@ cc_library( + "numerics/clamped_math.h", + "numerics/clamped_math_impl.h", + "numerics/safe_conversions.h", ++ "numerics/safe_conversions_arm_impl.h", + "numerics/safe_conversions_impl.h", + "numerics/safe_math_clang_gcc_impl.h", + "numerics/safe_math_shared_impl.h", +@@ -55,6 +56,7 @@ cc_library( "strings/string_number_conversions.h", "strings/utf_string_conversions.h", "strings/utf_string_conversion_utils.h", From 60317d23ac4cc4b0a6b1e2e4f334211aadbcdef9 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 3 Nov 2022 10:34:48 -0400 Subject: [PATCH 094/112] hot_restart: removing the proto from the no-hot-restart build (#23779) Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- envoy/server/BUILD | 1 - envoy/server/hot_restart.h | 2 -- source/server/BUILD | 3 +-- source/server/hot_restarting_base.h | 1 + test/mocks/server/hot_restart.h | 1 - test/mocks/server/instance.h | 1 - 6 files changed, 2 insertions(+), 7 deletions(-) diff --git a/envoy/server/BUILD b/envoy/server/BUILD index 77eeef992b6a..2c8f3e41a84b 100644 --- a/envoy/server/BUILD +++ b/envoy/server/BUILD @@ -109,7 +109,6 @@ envoy_cc_library( deps = [ "//envoy/event:dispatcher_interface", "//envoy/thread:thread_interface", - "//source/server:hot_restart_cc_proto", ], ) diff --git a/envoy/server/hot_restart.h b/envoy/server/hot_restart.h index aecf84a0c1b6..724af6c7f735 100644 --- a/envoy/server/hot_restart.h +++ b/envoy/server/hot_restart.h @@ -9,8 +9,6 @@ #include "envoy/stats/store.h" #include "envoy/thread/thread.h" -#include "source/server/hot_restart.pb.h" - namespace Envoy { namespace Server { diff --git a/source/server/BUILD b/source/server/BUILD index a1d7ac686fa0..4e76d1bc8a74 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -291,7 +291,6 @@ envoy_cc_library( srcs = envoy_select_hot_restart(["hot_restarting_base.cc"]), hdrs = envoy_select_hot_restart(["hot_restarting_base.h"]), deps = [ - ":hot_restart_cc_proto", "//envoy/api:os_sys_calls_interface", "//envoy/event:dispatcher_interface", "//envoy/event:file_event_interface", @@ -306,7 +305,7 @@ envoy_cc_library( "//source/common/common:utility_lib", "//source/common/network:utility_lib", "//source/common/stats:utility_lib", - ], + ] + envoy_select_hot_restart([":hot_restart_cc_proto"]), ) envoy_cc_library( diff --git a/source/server/hot_restarting_base.h b/source/server/hot_restarting_base.h index 16eefe939282..74db3b702d26 100644 --- a/source/server/hot_restarting_base.h +++ b/source/server/hot_restarting_base.h @@ -16,6 +16,7 @@ #include "envoy/stats/scope.h" #include "source/common/common/assert.h" +#include "source/server/hot_restart.pb.h" namespace Envoy { namespace Server { diff --git a/test/mocks/server/hot_restart.h b/test/mocks/server/hot_restart.h index 09049b87fdb1..8c6632bd92e4 100644 --- a/test/mocks/server/hot_restart.h +++ b/test/mocks/server/hot_restart.h @@ -17,7 +17,6 @@ class MockHotRestart : public HotRestart { MOCK_METHOD(void, drainParentListeners, ()); MOCK_METHOD(int, duplicateParentListenSocket, (const std::string& address, uint32_t worker_index)); - MOCK_METHOD(std::unique_ptr, getParentStats, ()); MOCK_METHOD(void, initialize, (Event::Dispatcher & dispatcher, Server::Instance& server)); MOCK_METHOD(absl::optional, sendParentAdminShutdownRequest, ()); MOCK_METHOD(void, sendParentTerminateRequest, ()); diff --git a/test/mocks/server/instance.h b/test/mocks/server/instance.h index a85008ebb01e..d4d0ae1afe80 100644 --- a/test/mocks/server/instance.h +++ b/test/mocks/server/instance.h @@ -59,7 +59,6 @@ class MockInstance : public Instance { MOCK_METHOD(DrainManager&, drainManager, ()); MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, ()); MOCK_METHOD(void, failHealthcheck, (bool fail)); - MOCK_METHOD(void, exportStatsToChild, (envoy::HotRestartMessage::Reply::Stats*)); MOCK_METHOD(bool, healthCheckFailed, ()); MOCK_METHOD(HotRestart&, hotRestart, ()); MOCK_METHOD(Init::Manager&, initManager, ()); From e0cc4306253f0318f9dcbc77cf32feb6245c378f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Thu, 3 Nov 2022 10:38:09 -0400 Subject: [PATCH 095/112] [redis] refactor and simplify onRedirection() (#23802) As part of #23764, we need to simplify what the Redis Client does after calling onRedirection(), so that eventually the ConnPool can cleanly handle async DNS lookups. Signed-off-by: Raul Gutierrez Segales --- .../extensions/clusters/redis/redis_cluster.h | 6 +-- .../filters/network/common/redis/client.h | 7 +-- .../network/common/redis/client_impl.cc | 19 ++----- .../network/redis_proxy/conn_pool_impl.cc | 30 ++++++----- .../network/redis_proxy/conn_pool_impl.h | 5 +- .../extensions/health_checkers/redis/redis.cc | 3 +- .../extensions/health_checkers/redis/redis.h | 2 +- .../clusters/redis/redis_cluster_test.cc | 2 +- .../network/common/redis/client_impl_test.cc | 20 ++----- .../filters/network/common/redis/mocks.h | 6 +-- .../redis_proxy/conn_pool_impl_test.cc | 53 ++++++++++++++----- 11 files changed, 79 insertions(+), 74 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index a71db06dc5a5..c8dc24cf6989 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -236,10 +236,8 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; void onFailure() override; // Note: Below callback isn't used in topology updates - bool onRedirection(NetworkFilters::Common::Redis::RespValuePtr&&, const std::string&, - bool) override { - return true; - } + void onRedirection(NetworkFilters::Common::Redis::RespValuePtr&&, const std::string&, + bool) override {} void onUnexpectedResponse(const NetworkFilters::Common::Redis::RespValuePtr&); Network::Address::InstanceConstSharedPtr diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index 9fc3e37d81c8..f9835be41238 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -50,9 +50,8 @@ class ClientCallbacks { * @param value supplies the MOVED error response * @param host_address supplies the redirection host address and port * @param ask_redirection indicates if this is a ASK redirection - * @return bool true if the request is successfully redirected, false otherwise */ - virtual bool onRedirection(RespValuePtr&& value, const std::string& host_address, + virtual void onRedirection(RespValuePtr&& value, const std::string& host_address, bool ask_redirection) PURE; }; @@ -65,9 +64,7 @@ class DoNothingPoolCallbacks : public ClientCallbacks { // ClientCallbacks void onResponse(Common::Redis::RespValuePtr&&) override {} void onFailure() override {} - bool onRedirection(Common::Redis::RespValuePtr&&, const std::string&, bool) override { - return false; - } + void onRedirection(Common::Redis::RespValuePtr&&, const std::string&, bool) override {} }; /** diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index e7279a676aea..bb3add9ab8a9 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -225,22 +225,13 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { } else if (config_.enableRedirection() && !is_transaction_client_ && (value->type() == Common::Redis::RespType::Error)) { std::vector err = StringUtil::splitToken(value->asString(), " ", false); - bool redirected = false; - if (err.size() == 3) { + if (err.size() == 3 && + (err[0] == RedirectionResponse::get().MOVED || err[0] == RedirectionResponse::get().ASK)) { // MOVED and ASK redirection errors have the following substrings: MOVED or ASK (err[0]), hash // key slot (err[1]), and IP address and TCP port separated by a colon (err[2]) - if (err[0] == RedirectionResponse::get().MOVED || err[0] == RedirectionResponse::get().ASK) { - redirected = true; - bool redirect_succeeded = callbacks.onRedirection(std::move(value), std::string(err[2]), - err[0] == RedirectionResponse::get().ASK); - if (redirect_succeeded) { - host_->cluster().stats().upstream_internal_redirect_succeeded_total_.inc(); - } else { - host_->cluster().stats().upstream_internal_redirect_failed_total_.inc(); - } - } - } - if (!redirected) { + callbacks.onRedirection(std::move(value), std::string(err[2]), + err[0] == RedirectionResponse::get().ASK); + } else { if (err[0] == RedirectionResponse::get().CLUSTER_DOWN) { callbacks.onFailure(); } else { diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index ce8c3ab8a552..917bddb56ec3 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -274,7 +274,7 @@ InstanceImpl::ThreadLocalPool::makeRequest(const std::string& key, RespVariant&& transaction.connection_established_ = true; } - pending_requests_.emplace_back(*this, std::move(request), callbacks); + pending_requests_.emplace_back(*this, std::move(request), callbacks, host); PendingRequest& pending_request = pending_requests_.back(); if (!transaction.active_) { @@ -399,9 +399,10 @@ void InstanceImpl::ThreadLocalActiveClient::onEvent(Network::ConnectionEvent eve InstanceImpl::PendingRequest::PendingRequest(InstanceImpl::ThreadLocalPool& parent, RespVariant&& incoming_request, - PoolCallbacks& pool_callbacks) + PoolCallbacks& pool_callbacks, + Upstream::HostConstSharedPtr& host) : parent_(parent), incoming_request_(std::move(incoming_request)), - pool_callbacks_(pool_callbacks) {} + pool_callbacks_(pool_callbacks), host_(host) {} InstanceImpl::PendingRequest::~PendingRequest() { if (request_handler_) { @@ -426,9 +427,12 @@ void InstanceImpl::PendingRequest::onFailure() { parent_.onRequestCompleted(); } -bool InstanceImpl::PendingRequest::onRedirection(Common::Redis::RespValuePtr&& value, +void InstanceImpl::PendingRequest::onRedirection(Common::Redis::RespValuePtr&& value, const std::string& host_address, bool ask_redirection) { + // This request might go away, so keep a copy of host. + auto host = host_; + // Prepend request with an asking command if redirected via an ASK error. The returned handle is // not important since there is no point in being able to cancel the request. The use of // null_pool_callbacks ensures the transparent filtering of the Redis server's response to the @@ -438,15 +442,17 @@ bool InstanceImpl::PendingRequest::onRedirection(Common::Redis::RespValuePtr&& v !parent_.makeRequestToHost(host_address, Common::Redis::Utility::AskingRequest::instance(), null_client_callbacks)) { onResponse(std::move(value)); - return false; - } - request_handler_ = parent_.makeRequestToHost(host_address, getRequest(incoming_request_), *this); - if (!request_handler_) { - onResponse(std::move(value)); - return false; + host->cluster().stats().upstream_internal_redirect_failed_total_.inc(); } else { - parent_.refresh_manager_->onRedirection(parent_.cluster_name_); - return true; + request_handler_ = + parent_.makeRequestToHost(host_address, getRequest(incoming_request_), *this); + if (!request_handler_) { + onResponse(std::move(value)); + host->cluster().stats().upstream_internal_redirect_failed_total_.inc(); + } else { + parent_.refresh_manager_->onRedirection(parent_.cluster_name_); + host->cluster().stats().upstream_internal_redirect_succeeded_total_.inc(); + } } } diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index be29c98ea026..2c55c5899ad4 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -106,13 +106,13 @@ class InstanceImpl : public Instance, public std::enable_shared_from_thistype(NetworkFilters::Common::Redis::RespType::Error); dummy_value->asString() = "dummy text"; - EXPECT_TRUE(discovery_session.onRedirection(std::move(dummy_value), "dummy ip", false)); + discovery_session.onRedirection(std::move(dummy_value), "dummy ip", false); RedisCluster::RedisDiscoveryClient discovery_client(discovery_session); EXPECT_NO_THROW(discovery_client.onAboveWriteBufferHighWatermark()); diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index f40b993ea578..bcd4d4e3810c 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -859,27 +859,21 @@ TEST_F(RedisClientImplTest, AskRedirection) { // The exact values of the hash slot and IP info are not important. response1->asString() = "ASK 1111 10.1.2.3:4321"; // Simulate redirection failure. - EXPECT_CALL(callbacks1, onRedirection_(Ref(response1), "10.1.2.3:4321", true)) - .WillOnce(Return(false)); + EXPECT_CALL(callbacks1, onRedirection_(Ref(response1), "10.1.2.3:4321", true)); EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::ExtOriginRequestSuccess, _)); callbacks_->onRespValue(std::move(response1)); - EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_internal_redirect_failed_total_.value()); - Common::Redis::RespValuePtr response2(new Common::Redis::RespValue()); response2->type(Common::Redis::RespType::Error); // The exact values of the hash slot and IP info are not important. response2->asString() = "ASK 2222 10.1.2.4:4321"; - EXPECT_CALL(callbacks2, onRedirection_(Ref(response2), "10.1.2.4:4321", true)) - .WillOnce(Return(true)); + EXPECT_CALL(callbacks2, onRedirection_(Ref(response2), "10.1.2.4:4321", true)); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::ExtOriginRequestSuccess, _)); callbacks_->onRespValue(std::move(response2)); - - EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_internal_redirect_succeeded_total_.value()); })); upstream_read_filter_->onData(fake_data, false); @@ -922,27 +916,21 @@ TEST_F(RedisClientImplTest, MovedRedirection) { // The exact values of the hash slot and IP info are not important. response1->asString() = "MOVED 1111 10.1.2.3:4321"; // Simulate redirection failure. - EXPECT_CALL(callbacks1, onRedirection_(Ref(response1), "10.1.2.3:4321", false)) - .WillOnce(Return(false)); + EXPECT_CALL(callbacks1, onRedirection_(Ref(response1), "10.1.2.3:4321", false)); EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::ExtOriginRequestSuccess, _)); callbacks_->onRespValue(std::move(response1)); - EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_internal_redirect_failed_total_.value()); - Common::Redis::RespValuePtr response2(new Common::Redis::RespValue()); response2->type(Common::Redis::RespType::Error); // The exact values of the hash slot and IP info are not important. response2->asString() = "MOVED 2222 10.1.2.4:4321"; - EXPECT_CALL(callbacks2, onRedirection_(Ref(response2), "10.1.2.4:4321", false)) - .WillOnce(Return(true)); + EXPECT_CALL(callbacks2, onRedirection_(Ref(response2), "10.1.2.4:4321", false)); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::ExtOriginRequestSuccess, _)); callbacks_->onRespValue(std::move(response2)); - - EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_internal_redirect_succeeded_total_.value()); })); upstream_read_filter_->onData(fake_data, false); diff --git a/test/extensions/filters/network/common/redis/mocks.h b/test/extensions/filters/network/common/redis/mocks.h index e79464e5baef..f0e8312c1c3e 100644 --- a/test/extensions/filters/network/common/redis/mocks.h +++ b/test/extensions/filters/network/common/redis/mocks.h @@ -99,14 +99,14 @@ class MockClientCallbacks : public ClientCallbacks { ~MockClientCallbacks() override; void onResponse(Common::Redis::RespValuePtr&& value) override { onResponse_(value); } - bool onRedirection(Common::Redis::RespValuePtr&& value, const std::string& host_address, + void onRedirection(Common::Redis::RespValuePtr&& value, const std::string& host_address, bool ask_redirection) override { - return onRedirection_(value, host_address, ask_redirection); + onRedirection_(value, host_address, ask_redirection); } MOCK_METHOD(void, onResponse_, (Common::Redis::RespValuePtr & value)); MOCK_METHOD(void, onFailure, ()); - MOCK_METHOD(bool, onRedirection_, + MOCK_METHOD(void, onRedirection_, (Common::Redis::RespValuePtr & value, const std::string& host_address, bool ask_redirection)); }; diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 4cf575f78f44..cac915c59232 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -285,8 +285,15 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client moved_response->type(Common::Redis::RespType::Error); moved_response->asString() = "MOVE 1111 " + host_address; EXPECT_CALL(callbacks, onResponse_(Ref(moved_response))); - EXPECT_FALSE(client->client_callbacks_.back()->onRedirection(std::move(moved_response), - host_address, false)); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, cluster()); + const auto expected = cm_.thread_local_cluster_.lb_.host_->cluster_.stats_store_ + .counter("upstream_internal_redirect_failed_total") + .value() + + 1; + client->client_callbacks_.back()->onRedirection(std::move(moved_response), host_address, false); + EXPECT_EQ(expected, cm_.thread_local_cluster_.lb_.host_->cluster_.stats_store_ + .counter("upstream_internal_redirect_failed_total") + .value()); } MOCK_METHOD(Common::Redis::Client::Client*, create_, (Upstream::HostConstSharedPtr host)); @@ -1085,9 +1092,13 @@ TEST_F(RedisConnPoolImplTest, MovedRedirectionSuccess) { EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host1), Return(client2))); EXPECT_CALL(*client2, makeRequest_(Ref(*request_value), _)).WillOnce(Return(&active_request2)); - EXPECT_TRUE(client->client_callbacks_.back()->onRedirection(std::move(moved_response), - "10.1.2.3:4000", false)); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, cluster()); + client->client_callbacks_.back()->onRedirection(std::move(moved_response), "10.1.2.3:4000", + false); EXPECT_EQ(host1->address()->asString(), "10.1.2.3:4000"); + EXPECT_EQ(1UL, cm_.thread_local_cluster_.lb_.host_->cluster_.stats_store_ + .counter("upstream_internal_redirect_succeeded_total") + .value()); respond(callbacks, client2); @@ -1136,8 +1147,9 @@ TEST_F(RedisConnPoolImplTest, MovedRedirectionFailure) { EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host1), Return(client2))); EXPECT_CALL(*client2, makeRequest_(Ref(*request3), _)).WillOnce(Return(nullptr)); EXPECT_CALL(callbacks, onResponse_(Ref(moved_response3))); - EXPECT_FALSE(client->client_callbacks_.back()->onRedirection(std::move(moved_response3), - "10.1.2.3:4000", false)); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, cluster()); + client->client_callbacks_.back()->onRedirection(std::move(moved_response3), "10.1.2.3:4000", + false); EXPECT_EQ(host1->address()->asString(), "10.1.2.3:4000"); EXPECT_CALL(*client, close()); @@ -1167,9 +1179,12 @@ TEST_F(RedisConnPoolImplTest, AskRedirectionSuccess) { EXPECT_CALL(*client2, makeRequest_(Ref(Common::Redis::Utility::AskingRequest::instance()), _)) .WillOnce(Return(&ask_request)); EXPECT_CALL(*client2, makeRequest_(Ref(*request_value), _)).WillOnce(Return(&active_request2)); - EXPECT_TRUE(client->client_callbacks_.back()->onRedirection(std::move(ask_response), - "10.1.2.3:4000", true)); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, cluster()); + client->client_callbacks_.back()->onRedirection(std::move(ask_response), "10.1.2.3:4000", true); EXPECT_EQ(host1->address()->asString(), "10.1.2.3:4000"); + EXPECT_EQ(1UL, cm_.thread_local_cluster_.lb_.host_->cluster_.stats_store_ + .counter("upstream_internal_redirect_succeeded_total") + .value()); respond(callbacks, client2); @@ -1198,9 +1213,12 @@ TEST_F(RedisConnPoolImplTest, AskRedirectionFailure) { EXPECT_CALL(*client2, makeRequest_(Ref(Common::Redis::Utility::AskingRequest::instance()), _)) .WillOnce(Return(nullptr)); EXPECT_CALL(callbacks, onResponse_(Ref(ask_response3))); - EXPECT_FALSE(client->client_callbacks_.back()->onRedirection(std::move(ask_response3), - "10.1.2.3:4000", true)); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, cluster()); + client->client_callbacks_.back()->onRedirection(std::move(ask_response3), "10.1.2.3:4000", true); EXPECT_EQ(host1->address()->asString(), "10.1.2.3:4000"); + EXPECT_EQ(1UL, cm_.thread_local_cluster_.lb_.host_->cluster_.stats_store_ + .counter("upstream_internal_redirect_failed_total") + .value()); // Test an upstream error from trying to send the original request after the "asking" command is // sent successfully. @@ -1214,8 +1232,11 @@ TEST_F(RedisConnPoolImplTest, AskRedirectionFailure) { .WillOnce(Return(&active_request5)); EXPECT_CALL(*client2, makeRequest_(Ref(*request4), _)).WillOnce(Return(nullptr)); EXPECT_CALL(callbacks, onResponse_(Ref(ask_response4))); - EXPECT_FALSE(client->client_callbacks_.back()->onRedirection(std::move(ask_response4), - "10.1.2.3:4000", true)); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, cluster()); + client->client_callbacks_.back()->onRedirection(std::move(ask_response4), "10.1.2.3:4000", true); + EXPECT_EQ(2UL, cm_.thread_local_cluster_.lb_.host_->cluster_.stats_store_ + .counter("upstream_internal_redirect_failed_total") + .value()); EXPECT_CALL(*client, close()); tls_.shutdownThread(); @@ -1267,9 +1288,13 @@ TEST_F(RedisConnPoolImplTest, MakeRequestAndRedirectFollowedByDelete) { EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host1), Return(client2))); EXPECT_CALL(*client2, makeRequest_(Ref(*value), _)).WillOnce(Return(&active_request2)); - EXPECT_TRUE(client->client_callbacks_.back()->onRedirection(std::move(moved_response), - "10.1.2.3:4000", false)); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, cluster()); + client->client_callbacks_.back()->onRedirection(std::move(moved_response), "10.1.2.3:4000", + false); EXPECT_EQ(host1->address()->asString(), "10.1.2.3:4000"); + EXPECT_EQ(1UL, cm_.thread_local_cluster_.lb_.host_->cluster_.stats_store_ + .counter("upstream_internal_redirect_succeeded_total") + .value()); EXPECT_CALL(callbacks, onResponse_(_)); client2->client_callbacks_.back()->onResponse(std::make_unique()); From d5bb627e4a37366ea7316bbc9f1614d680b079e8 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 3 Nov 2022 14:39:15 +0000 Subject: [PATCH 096/112] repo: Improve key signing of binaries (#23640) Signed-off-by: Ryan Northey --- .azure-pipelines/pipelines.yml | 11 +++++++++-- ci/publish_github_assets.sh | 25 +++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 73928d325aa4..4c5b22abc2c2 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -542,6 +542,13 @@ stages: itemPattern: "bazel.release.arm64/envoy-contrib_binary.tar.gz" downloadType: single targetPath: $(Build.StagingDirectory) + + - task: DownloadSecureFile@1 + name: MaintainerGPGKey + displayName: 'Download maintainer GPG key' + inputs: + secureFile: 'maintainer.gpg.key' + - bash: | set -e @@ -561,13 +568,13 @@ stages: cp -a linux/arm64/build_envoy_release_stripped/envoy "publish/envoy-${VERSION}-linux-aarch_64" cp -a linux/arm64/build_envoy-contrib_release_stripped/envoy "publish/envoy-contrib-${VERSION}-linux-aarch_64" - echo "$MAINTAINER_GPG_KEY" | base64 -d | gpg --import - + echo "$MAINTAINER_GPG_KEY_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --import "$(MaintainerGPGKey.secureFilePath)" ci/publish_github_assets.sh "v${VERSION}" "${PWD}/publish" workingDirectory: $(Build.SourcesDirectory) env: GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken) - MAINTAINER_GPG_KEY: $(MaintainerGPGKey) + MAINTAINER_GPG_KEY_PASSPHRASE: $(MaintainerGPGKeyPassphrase) - stage: verify dependsOn: ["docker"] diff --git a/ci/publish_github_assets.sh b/ci/publish_github_assets.sh index 22c4b62208e1..26dfd3b660d8 100755 --- a/ci/publish_github_assets.sh +++ b/ci/publish_github_assets.sh @@ -6,7 +6,23 @@ PUBLISH_DIR="$2" REPO_OWNER="${REPO_OWNER:-envoyproxy}" REPO_NAME="${REPO_NAME:-envoy}" RELEASE_API_URL="https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/releases" +MAINTAINER_GPG_KEY_PASSPHRASE="${MAINTAINER_GPG_KEY_PASSPHRASE:-}" +GITHUB_TOKEN="${GITHUB_TOKEN:-}" +if [[ -z "$GITHUB_TOKEN" ]]; then + # shellcheck disable=SC2016 + echo 'env var `GITHUB_TOKEN` must be set' + exit 1 +fi + + +gpg_sign () { + if [[ -n "$MAINTAINER_GPG_KEY_PASSPHRASE" ]]; then + echo "$MAINTAINER_GPG_KEY_PASSPHRASE" | gpg --pinentry-mode loopback --passphrase-fd 0 --clearsign checksums.txt + else + gpg --clearsign checksums.txt + fi +} sign_assets () { local asset @@ -21,7 +37,7 @@ sign_assets () { sha256sum "$asset" >> "checksums.txt" done - gpg --clearsign checksums.txt + gpg_sign checksums.txt rm checksums.txt cat checksums.txt.asc } @@ -66,11 +82,16 @@ upload_to_github () { upload_assets () { local release_id upload_url + release_id="$(get_release_id "${1}")" + if [[ "$release_id" == null ]]; then + # shellcheck disable=SC2016 + echo 'Failed querying github API - `GITHUB_TOKEN` may not be valid or the release ('"${release_id}"') was not found' + return 1 + fi upload_url="$(get_upload_url "$release_id")" echo "Upload assets (${PUBLISH_DIR}) -> ${upload_url}" - for asset in ./*; do asset="$(echo "${asset}" | cut -d/ -f2)" upload_to_github "${upload_url}" "$asset" From 35bf11a0f14a996d4e95b5302b0a953e0beba9f6 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 3 Nov 2022 14:40:08 +0000 Subject: [PATCH 097/112] repo: Add yaml linting (#23708) Signed-off-by: Ryan Northey --- tools/base/requirements.in | 4 ++-- tools/base/requirements.txt | 25 ++++++++++++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/tools/base/requirements.in b/tools/base/requirements.in index e6f51104ea3f..a88409ba78e2 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -4,8 +4,8 @@ aiohttp>=3.8.1 cffi>=1.15.0 colorama coloredlogs -envoy.base.utils>=0.3.7 -envoy.code.check>=0.3.0 +envoy.base.utils>=0.3.8 +envoy.code.check>=0.3.3 envoy.dependency.check>=0.1.5 envoy.dependency.pip_check>=0.1.2 envoy.distribution.release>=0.0.8 diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 368c1873ca49..e5b5614c5389 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -324,9 +324,9 @@ docutils==0.16 \ # sphinx # sphinx-rtd-theme # sphinx-tabs -envoy-base-utils==0.3.7 \ - --hash=sha256:5c79aa70de6b7014a5b955822eb83324fd222b7cf7bb656e12a018c747b4f2df \ - --hash=sha256:ac7850ec1c90253964a5d7f7847a3c3317f9d77607e6f90d1fb5aeb975ac2842 +envoy-base-utils==0.3.8 \ + --hash=sha256:9c4190bae708a75a955a7c09f9a674121c92dd7e33a359de3715332651ffe2bb \ + --hash=sha256:ae44642f2f43ac778f854f41c931d1969fe96dc323e9112876bc02b94fda05b2 # via # -r requirements.in # envoy-code-check @@ -339,9 +339,9 @@ envoy-base-utils==0.3.7 \ # envoy-docs-sphinx-runner # envoy-github-release # envoy-gpg-sign -envoy-code-check==0.3.0 \ - --hash=sha256:9383b4fcfcb22d6733813b3a75c9e40b1ba62f5a13c11b7e3366c31a27e4057a \ - --hash=sha256:f23069e123770e5b3a4e4e050c01d03f6c87144e09f9a20f488ce6768eda5f4c +envoy-code-check==0.3.3 \ + --hash=sha256:4483afa45d58d9a00332a281dc1e49f7942fe27027acc2a8f490ee79446c2125 \ + --hash=sha256:d520feae0470756c77a4b562a92d3d4f8bc5a7ef7b195598f9c62170bd58b588 # via -r requirements.in envoy-dependency-check==0.1.5 \ --hash=sha256:2394626236e7b43c0273d4a0a43e1e13d3990ebde9ebdacea253464000927961 \ @@ -716,6 +716,10 @@ packaging==21.0 \ # envoy-github-abstract # envoy-github-release # sphinx +pathspec==0.10.1 \ + --hash=sha256:46846318467efc4556ccfd27816e004270a9eeeeb4d062ce5e6fc7a87c573f93 \ + --hash=sha256:7ace6161b621d31e7902eb6b5ae148d12cfd23f4a249b9ffb6b9fee12084323d + # via yamllint pep8-naming==0.13.2 \ --hash=sha256:59e29e55c478db69cffbe14ab24b5bd2cd615c0413edf790d47d3fb7ba9a4e23 \ --hash=sha256:93eef62f525fd12a6f8c98f4dcc17fa70baae2f37fa1f73bec00e3e44392fa48 @@ -846,6 +850,7 @@ pyyaml==6.0 \ # -r requirements.in # envoy-base-utils # envoy-code-check + # yamllint requests==2.26.0 \ --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \ --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7 @@ -987,6 +992,10 @@ verboselogs==1.7 \ wrapt==1.12.1 \ --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7 # via deprecated +yamllint==1.28.0 \ + --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ + --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b + # via envoy-code-check yapf==0.32.0 \ --hash=sha256:8fea849025584e486fd06d6ba2bed717f396080fd3cc236ba10cb97c4c51cf32 \ --hash=sha256:a3f5085d37ef7e3e004c4ba9f9b3e40c54ff1901cd111f05145ae313a7c67d1b @@ -1061,4 +1070,6 @@ yarl==1.8.1 \ setuptools==65.5.0 \ --hash=sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17 \ --hash=sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356 - # via -r requirements.in + # via + # -r requirements.in + # yamllint From 74753956f63d9726da81418e26a9a176b88363a8 Mon Sep 17 00:00:00 2001 From: Andy Fong Date: Thu, 3 Nov 2022 13:36:04 -0400 Subject: [PATCH 098/112] http: fixed Utility::PercentEncoding::encode() to treat all values as unsigned (#23781) * http: fixed Utility::PercentEncoding::encode() to treat all values as unsigned Signed-off-by: Andy Fong --- changelogs/current.yaml | 3 +++ source/common/http/utility.cc | 2 +- test/common/http/utility_test.cc | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 54de9a576177..b38ee6f293e7 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -29,6 +29,9 @@ bug_fixes: - area: generic_proxy change: | fixed a bug that encoder filters and decoder filters of generic proxy will be executed in the same order. The encoder filters' execuate order should be the reverse of decoder filters' in the generic proxy. +- area: http + change: | + fixed a bug where Utility::PercentEncoding::encode() encodes some characters incorrectly because it was treating the value as negative. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc index 498ba56cf96a..e5d8b851f8bc 100644 --- a/source/common/http/utility.cc +++ b/source/common/http/utility.cc @@ -1006,7 +1006,7 @@ std::string Utility::PercentEncoding::encode(absl::string_view value, const size if (ch < ' ' || ch >= '~' || reserved_char_set.find(ch) != reserved_char_set.end()) { // For consistency, URI producers should use uppercase hexadecimal digits for all // percent-encodings. https://tools.ietf.org/html/rfc3986#section-2.1. - absl::StrAppend(&encoded, fmt::format("%{:02X}", ch)); + absl::StrAppend(&encoded, fmt::format("%{:02X}", static_cast(ch))); } else { encoded.push_back(ch); } diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index cf3115c9c268..1794081394e3 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -1633,6 +1633,7 @@ TEST(PercentEncoding, Encoding) { EXPECT_EQ(Utility::PercentEncoding::encode("too%large"), "too%25large"); EXPECT_EQ(Utility::PercentEncoding::encode("too%!large/"), "too%25!large/"); EXPECT_EQ(Utility::PercentEncoding::encode("too%!large/", "%!/"), "too%25%21large%2F"); + EXPECT_EQ(Utility::PercentEncoding::encode("São Paulo"), "S%C3%A3o Paulo"); } TEST(CheckRequiredHeaders, Request) { From 02586f1c7d7525037bf62b1a045ecd4c632b2aaa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Nov 2022 13:57:03 -0400 Subject: [PATCH 099/112] build(deps): bump github/codeql-action from 2.1.29 to 2.1.30 (#23801) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.1.29 to 2.1.30. Signed-off-by: dependabot[bot] --- .github/workflows/codeql-daily.yml | 4 ++-- .github/workflows/codeql-push.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index be3fcf3fc4aa..3b70679003c3 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -28,7 +28,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 + uses: github/codeql-action/init@18fe527fa8b29f134bb91f32f1a5dc5abb15ed7f # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -55,4 +55,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 + uses: github/codeql-action/analyze@18fe527fa8b29f134bb91f32f1a5dc5abb15ed7f diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index 32493d144a8c..495a36ed9e58 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -41,7 +41,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 + uses: github/codeql-action/init@18fe527fa8b29f134bb91f32f1a5dc5abb15ed7f # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -71,4 +71,4 @@ jobs: - name: Perform CodeQL Analysis if: env.BUILD_TARGETS != '' - uses: github/codeql-action/analyze@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6 + uses: github/codeql-action/analyze@18fe527fa8b29f134bb91f32f1a5dc5abb15ed7f From d8ba42bfc7506b67640c879fa2d3783ff6ef0282 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Nov 2022 14:03:26 -0400 Subject: [PATCH 100/112] build(deps): bump golang from 1.19.2-bullseye to 1.19.3-bullseye in /examples/grpc-bridge/server (#23804) build(deps): bump golang in /examples/grpc-bridge/server Bumps golang from 1.19.2-bullseye to 1.19.3-bullseye. Signed-off-by: dependabot[bot] --- examples/grpc-bridge/server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/grpc-bridge/server/Dockerfile b/examples/grpc-bridge/server/Dockerfile index 493de4907eda..f1e98fd33f3c 100644 --- a/examples/grpc-bridge/server/Dockerfile +++ b/examples/grpc-bridge/server/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.2-bullseye@sha256:8b9971c37678d2c4c04e7dd4e430baec049647d72ed97bf5e6a41ef8e77e74a5 as builder +FROM golang:1.19.3-bullseye@sha256:34e901ebac66df44ce97b56a9e1bb407307e54fe13e843d6c59da7826ce4dd2c as builder WORKDIR /build From 5f5c4b05d0f31e5b0031c861bc798c0e592e71fc Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 3 Nov 2022 19:21:49 +0000 Subject: [PATCH 101/112] deps: Bump `net_zlib` -> 1.2.13 (#23796) Fix #23511 Signed-off-by: Ryan Northey --- bazel/foreign_cc/zlib.patch | 50 +++++++++++++++------------------- bazel/repository_locations.bzl | 6 ++-- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/bazel/foreign_cc/zlib.patch b/bazel/foreign_cc/zlib.patch index aeeeed3147fd..64484e08f7ff 100644 --- a/bazel/foreign_cc/zlib.patch +++ b/bazel/foreign_cc/zlib.patch @@ -1,70 +1,72 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index e108c16..2cd82ef 100644 +index b412dc7..658a109 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -183,10 +183,18 @@ if(MINGW) +@@ -147,10 +147,15 @@ if(MINGW) set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj) endif(MINGW) - --add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) --add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) + +-add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) +-add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) -set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) -set_target_properties(zlib PROPERTIES SOVERSION 1) +if(NOT DEFINED BUILD_SHARED_LIBS) -+ add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) -+ add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) ++ add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) ++ add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) + set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) + set_target_properties(zlib PROPERTIES SOVERSION 1) -+ -+ set(ZLIB_INSTALL_LIBRARIES zlib zlibstatic) +else() + add_library(zlib ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) -+ + set(ZLIB_INSTALL_LIBRARIES zlib) +endif() - + if(NOT CYGWIN) # This property causes shared libraries on Linux to have the full version -@@ -196,22 +204,22 @@ if(NOT CYGWIN) +@@ -160,22 +165,22 @@ if(NOT CYGWIN) # # This has no effect with MSVC, on that platform the version info for # the DLL comes from the resource file win32/zlib1.rc - set_target_properties(zlib PROPERTIES VERSION ${ZLIB_FULL_VERSION}) + set_target_properties(${ZLIB_INSTALL_LIBRARIES} PROPERTIES VERSION ${ZLIB_FULL_VERSION}) endif() - + if(UNIX) # On unix-like platforms the library is almost always called libz - set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z) + set_target_properties(${ZLIB_INSTALL_LIBRARIES} PROPERTIES OUTPUT_NAME z) if(NOT APPLE) - set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") -+ set_target_properties(${ZLIB_INSTALL_LIBRARIES} PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") ++ set_target_properties(${ZLIB_INSTALL_LIBRARIES} PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") endif() elseif(BUILD_SHARED_LIBS AND WIN32) # Creates zlib1.dll when building shared library version - set_target_properties(zlib PROPERTIES SUFFIX "1.dll") + set_target_properties(${ZLIB_INSTALL_LIBRARIES} PROPERTIES SUFFIX "1.dll") endif() - + if(NOT SKIP_INSTALL_LIBRARIES AND NOT SKIP_INSTALL_ALL ) - install(TARGETS zlib zlibstatic + install(TARGETS ${ZLIB_INSTALL_LIBRARIES} RUNTIME DESTINATION "${INSTALL_BIN_DIR}" ARCHIVE DESTINATION "${INSTALL_LIB_DIR}" LIBRARY DESTINATION "${INSTALL_LIB_DIR}" ) -@@ -229,21 +237,22 @@ endif() - #============================================================================ +@@ -194,20 +199,22 @@ endif() # Example binaries #============================================================================ -- + -add_executable(example test/example.c) -target_link_libraries(example zlib) -add_test(example example) -- ++if(NOT SKIP_BUILD_EXAMPLES) ++ add_executable(example test/example.c) ++ target_link_libraries(example zlib) ++ add_test(example example) + -add_executable(minigzip test/minigzip.c) -target_link_libraries(minigzip zlib) -- ++ add_executable(minigzip test/minigzip.c) ++ target_link_libraries(minigzip zlib) + -if(HAVE_OFF64_T) - add_executable(example64 test/example.c) - target_link_libraries(example64 zlib) @@ -74,14 +76,6 @@ index e108c16..2cd82ef 100644 - add_executable(minigzip64 test/minigzip.c) - target_link_libraries(minigzip64 zlib) - set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") -+if(NOT SKIP_BUILD_EXAMPLES) -+ add_executable(example test/example.c) -+ target_link_libraries(example zlib) -+ add_test(example example) -+ -+ add_executable(minigzip test/minigzip.c) -+ target_link_libraries(minigzip zlib) -+ + if(HAVE_OFF64_T) + add_executable(example64 test/example.c) + target_link_libraries(example64 zlib) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 93f9763fda83..79a681c9bbdf 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -635,12 +635,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "zlib", project_desc = "zlib compression library", project_url = "https://zlib.net", - version = "1.2.12", - sha256 = "d8688496ea40fb61787500e863cc63c9afcbc524468cedeb478068924eb54932", + version = "1.2.13", + sha256 = "1525952a0a567581792613a9723333d7f8cc20b87a81f920fb8bc7e3f2251428", strip_prefix = "zlib-{version}", urls = ["https://github.com/madler/zlib/archive/v{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2022-03-27", + release_date = "2022-10-14", cpe = "cpe:2.3:a:gnu:zlib:*", license = "zlib", license_url = "https://github.com/madler/zlib/blob/v{version}/zlib.h", From b65a09354be531be7a5928df2f268c6285e9a261 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 3 Nov 2022 19:22:40 +0000 Subject: [PATCH 102/112] deps: Bump `com_github_unicode_org_icu` -> 72.1 (#23794) Fix #23552 Signed-off-by: Ryan Northey --- bazel/foreign_cc/icu.patch | 44 +++++++++++++++++++++++++++------- bazel/repository_locations.bzl | 6 ++--- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/bazel/foreign_cc/icu.patch b/bazel/foreign_cc/icu.patch index 83ffc7f1a5f2..1ad17d9024de 100644 --- a/bazel/foreign_cc/icu.patch +++ b/bazel/foreign_cc/icu.patch @@ -1,6 +1,6 @@ diff --git a/icu4c/source/common/BUILD.bazel b/icu4c/source/common/BUILD.bazel deleted file mode 100644 -index e385d3b..0000000 +index e385d3b243..0000000000 --- a/icu4c/source/common/BUILD.bazel +++ /dev/null @@ -1,1213 +0,0 @@ @@ -1219,7 +1219,7 @@ index e385d3b..0000000 -) diff --git a/icu4c/source/data/unidata/norm2/BUILD.bazel b/icu4c/source/data/unidata/norm2/BUILD.bazel deleted file mode 100644 -index 049e19b..0000000 +index 049e19bd41..0000000000 --- a/icu4c/source/data/unidata/norm2/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ @@ -1238,7 +1238,7 @@ index 049e19b..0000000 -]) diff --git a/icu4c/source/i18n/BUILD.bazel b/icu4c/source/i18n/BUILD.bazel deleted file mode 100644 -index 2d85cdb..0000000 +index 2d85cdb180..0000000000 --- a/icu4c/source/i18n/BUILD.bazel +++ /dev/null @@ -1,130 +0,0 @@ @@ -1373,9 +1373,26 @@ index 2d85cdb..0000000 - ], -) diff --git a/icu4c/source/icudefs.mk.in b/icu4c/source/icudefs.mk.in -index 2c35816..4ad6f52 100644 +index 2c358167a8..b30f33b292 100644 --- a/icu4c/source/icudefs.mk.in +++ b/icu4c/source/icudefs.mk.in +@@ -50,13 +50,13 @@ SO_TARGET_VERSION_MAJOR = @LIB_VERSION_MAJOR@ + # The ICU data external name is usually icudata; the entry point name is + # the version-dependent name (for no particular reason except it was easier + # to change the build this way). When building in common mode, the data +-# name is the versioned platform-dependent one. ++# name is the versioned platform-dependent one. + + ICUDATA_DIR = @pkgicudatadir@/$(PACKAGE)$(ICULIBSUFFIX)/$(VERSION) + + ICUDATA_BASENAME_VERSION = $(ICUPREFIX)dt@LIB_VERSION_MAJOR@ +-# the entry point is almost like the basename, but has the lib suffix. +-ICUDATA_ENTRY_POINT = $(ICUPREFIX)dt@ICULIBSUFFIXCNAME@@LIB_VERSION_MAJOR@ ++# the entry point is almost like the basename, but has the lib suffix. ++ICUDATA_ENTRY_POINT = $(ICUPREFIX)dt@ICULIBSUFFIXCNAME@@LIB_VERSION_MAJOR@ + ICUDATA_CHAR = @ICUDATA_CHAR@ + ICUDATA_PLATFORM_NAME = $(ICUDATA_BASENAME_VERSION)$(ICUDATA_CHAR) + PKGDATA_LIBSTATICNAME = -L $(STATIC_PREFIX)$(ICUPREFIX)$(DATA_STUBNAME)$(ICULIBSUFFIX) @@ -117,7 +117,7 @@ EXEEXT = @EXEEXT@ CC = @CC@ CXX = @CXX@ @@ -1385,12 +1402,21 @@ index 2c35816..4ad6f52 100644 RANLIB = @RANLIB@ COMPILE_LINK_ENVVAR = @COMPILE_LINK_ENVVAR@ UCLN_NO_AUTO_CLEANUP = @UCLN_NO_AUTO_CLEANUP@ +@@ -215,7 +215,7 @@ LIBICU = $(LIBPREFIX)$(ICUPREFIX) + ifneq ($(ENABLE_SHARED),YES) + STATIC_PREFIX_WHEN_USED = s + else +-STATIC_PREFIX_WHEN_USED = ++STATIC_PREFIX_WHEN_USED = + endif + + # Static library prefix and file extension diff --git a/icu4c/source/stubdata/BUILD.bazel b/icu4c/source/stubdata/BUILD.bazel deleted file mode 100644 -index abb7351..0000000 +index 20344ef499..0000000000 --- a/icu4c/source/stubdata/BUILD.bazel +++ /dev/null -@@ -1,23 +0,0 @@ +@@ -1,24 +0,0 @@ -# © 2021 and later: Unicode, Inc. and others. -# License & terms of use: http://www.unicode.org/copyright.html - @@ -1409,6 +1435,7 @@ index abb7351..0000000 -cc_library( - name = "stubdata", - srcs = ["stubdata.cpp"], +- hdrs = ["stubdata.h"], - deps = ["//icu4c/source/common:headers"], - local_defines = [ - "U_COMMON_IMPLEMENTATION", @@ -1416,7 +1443,7 @@ index abb7351..0000000 -) diff --git a/icu4c/source/tools/gennorm2/BUILD.bazel b/icu4c/source/tools/gennorm2/BUILD.bazel deleted file mode 100644 -index c602897..0000000 +index c602897baf..0000000000 --- a/icu4c/source/tools/gennorm2/BUILD.bazel +++ /dev/null @@ -1,39 +0,0 @@ @@ -1461,7 +1488,7 @@ index c602897..0000000 -) diff --git a/icu4c/source/tools/toolutil/BUILD.bazel b/icu4c/source/tools/toolutil/BUILD.bazel deleted file mode 100644 -index 276c857..0000000 +index 276c857f12..0000000000 --- a/icu4c/source/tools/toolutil/BUILD.bazel +++ /dev/null @@ -1,126 +0,0 @@ @@ -1591,4 +1618,3 @@ index 276c857..0000000 - "//icu4c/source/i18n:headers", - ], -) - diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 79a681c9bbdf..0a9e62b069d8 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -378,13 +378,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "ICU Library", project_desc = "Development files for International Components for Unicode", project_url = "https://github.com/unicode-org/icu", - version = "71-1", - sha256 = "d88a4ea7a4a28b445bb073a6cfeb2a296bf49a4a2fe5f1b49f87ecb4fc55c51d", + version = "72-1", + sha256 = "43cbad628d98f37a3f95f6c34579f9144ef4bde60248fa6004a4f006d7487e69", strip_prefix = "icu-release-{version}", urls = ["https://github.com/unicode-org/icu/archive/release-{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.http.language"], - release_date = "2022-04-06", + release_date = "2022-10-18", cpe = "N/A", license = "ICU", license_url = "https://github.com/unicode-org/icu/blob/release-{version}/icu4c/LICENSE", From 9c1980af8bd5268a94d5e3a1e992e4b91ec7f6fc Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 3 Nov 2022 16:22:09 -0400 Subject: [PATCH 103/112] googleurl: further fixes (#23820) fixing one more googleurl dep Signed-off-by: Alyssa Wilk --- bazel/external/googleurl.patch | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bazel/external/googleurl.patch b/bazel/external/googleurl.patch index a426657bee04..442ec4e9290b 100644 --- a/bazel/external/googleurl.patch +++ b/bazel/external/googleurl.patch @@ -123,11 +123,12 @@ diff --git a/base/BUILD b/base/BUILD index 60ca578..c922b73 100644 --- a/base/BUILD +++ b/base/BUILD -@@ -40,6 +40,7 @@ cc_library( +@@ -40,6 +40,8 @@ cc_library( "numerics/clamped_math.h", "numerics/clamped_math_impl.h", "numerics/safe_conversions.h", + "numerics/safe_conversions_arm_impl.h", ++ "numerics/safe_math_arm_impl.h", "numerics/safe_conversions_impl.h", "numerics/safe_math_clang_gcc_impl.h", "numerics/safe_math_shared_impl.h", From d93afd72e5b861ea8edcd97ebc26417d61f758c3 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 3 Nov 2022 15:12:48 -0700 Subject: [PATCH 104/112] ecds: deprecate top-level stats guard (#23611) Signed-off-by: Kuat Yessenov kuat@google.com Commit Message: Deprecate and remove stats prefix guard for ECDS. Additional Description: Risk Level: Testing: Docs Changes: Release Notes: yes Fixes: #23604 --- changelogs/current.yaml | 3 ++ envoy/filter/config_provider_manager.h | 4 +- source/common/filter/config_discovery_impl.h | 15 ++----- source/common/http/filter_chain_helper.h | 4 +- source/common/runtime/runtime_features.cc | 1 - source/server/listener_manager_impl.cc | 4 +- test/common/filter/BUILD | 1 - .../filter/config_discovery_impl_test.cc | 39 +++---------------- 8 files changed, 18 insertions(+), 53 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index b38ee6f293e7..5af5e50f782c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -62,6 +62,9 @@ removed_config_or_runtime: change: | removed ``envoy.reloadable_features.http_100_continue_case_insensitive`` and legacy code paths. removed ``envoy.reloadable_features.override_request_timeout_by_gateway_timeout`` and legacy code paths. +- area: ecds + change: | + removed ``envoy.reloadable_features.top_level_ecds_stats`` and legacy code paths. new_features: - area: build diff --git a/envoy/filter/config_provider_manager.h b/envoy/filter/config_provider_manager.h index af853904e5db..e9315b33f898 100644 --- a/envoy/filter/config_provider_manager.h +++ b/envoy/filter/config_provider_manager.h @@ -37,7 +37,6 @@ template class FilterConfigProviderManager { * @param config_source supplies the extension configuration source for the filter configs. * @param filter_config_name the filter config resource name. * @param factory_context is the context to use for the filter config provider. - * @param stat_prefix supplies the stat_prefix to use for the provider stats. * @param last_filter_in_filter_chain indicates whether this filter is the last filter in the * configured chain * @param filter_chain_type is the filter chain type @@ -48,8 +47,7 @@ template class FilterConfigProviderManager { const envoy::config::core::v3::ExtensionConfigSource& config_source, const std::string& filter_config_name, Server::Configuration::ServerFactoryContext& server_context, FactoryCtx& factory_context, - const std::string& stat_prefix, bool last_filter_in_filter_chain, - const std::string& filter_chain_type, + bool last_filter_in_filter_chain, const std::string& filter_chain_type, const Network::ListenerFilterMatcherSharedPtr& listener_filter_matcher) PURE; /** diff --git a/source/common/filter/config_discovery_impl.h b/source/common/filter/config_discovery_impl.h index 0d933789b999..a6678458c18f 100644 --- a/source/common/filter/config_discovery_impl.h +++ b/source/common/filter/config_discovery_impl.h @@ -405,20 +405,13 @@ class FilterConfigProviderManagerImpl : public FilterConfigProviderManagerImplBa const envoy::config::core::v3::ExtensionConfigSource& config_source, const std::string& filter_config_name, Server::Configuration::ServerFactoryContext& server_context, FactoryCtx& factory_context, - const std::string& stat_prefix, bool last_filter_in_filter_chain, - const std::string& filter_chain_type, + bool last_filter_in_filter_chain, const std::string& filter_chain_type, const Network::ListenerFilterMatcherSharedPtr& listener_filter_matcher) override { std::string subscription_stat_prefix; absl::string_view provider_stat_prefix; - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.top_level_ecds_stats")) { - subscription_stat_prefix = - absl::StrCat("extension_config_discovery.", statPrefix(), filter_config_name, "."); - provider_stat_prefix = subscription_stat_prefix; - } else { - subscription_stat_prefix = - absl::StrCat(stat_prefix, "extension_config_discovery.", filter_config_name, "."); - provider_stat_prefix = stat_prefix; - } + subscription_stat_prefix = + absl::StrCat("extension_config_discovery.", statPrefix(), filter_config_name, "."); + provider_stat_prefix = subscription_stat_prefix; auto subscription = getSubscription(config_source.config_source(), filter_config_name, server_context, subscription_stat_prefix); diff --git a/source/common/http/filter_chain_helper.h b/source/common/http/filter_chain_helper.h index 4ecf2bec6ed2..54c7d6b9f091 100644 --- a/source/common/http/filter_chain_helper.h +++ b/source/common/http/filter_chain_helper.h @@ -145,8 +145,8 @@ class FilterChainHelper : Logger::Loggable { } auto filter_config_provider = filter_config_provider_manager_.createDynamicFilterConfigProvider( - config_discovery, name, server_context_, factory_context_, stats_prefix_, - last_filter_in_current_config, filter_chain_type, nullptr); + config_discovery, name, server_context_, factory_context_, last_filter_in_current_config, + filter_chain_type, nullptr); filter_factories.push_back(std::move(filter_config_provider)); } diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 4b3cce5d327f..e1742fa3b20d 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -64,7 +64,6 @@ RUNTIME_GUARD(envoy_reloadable_features_skip_dns_lookup_for_proxied_requests); RUNTIME_GUARD(envoy_reloadable_features_test_feature_true); RUNTIME_GUARD(envoy_reloadable_features_thrift_allow_negative_field_ids); RUNTIME_GUARD(envoy_reloadable_features_tls_async_cert_validation); -RUNTIME_GUARD(envoy_reloadable_features_top_level_ecds_stats); RUNTIME_GUARD(envoy_reloadable_features_udp_proxy_connect); RUNTIME_GUARD(envoy_reloadable_features_unified_header_formatter); RUNTIME_GUARD(envoy_reloadable_features_use_rfc_connect); diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 9fa6bfb74a20..3a05196d785c 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -144,8 +144,8 @@ ProdListenerComponentFactory::createListenerFilterFactoryListImpl( } } auto filter_config_provider = config_provider_manager.createDynamicFilterConfigProvider( - config_discovery, name, context.getServerFactoryContext(), context, "tcp_listener.", - false, "listener", createListenerFilterMatcher(proto_config)); + config_discovery, name, context.getServerFactoryContext(), context, false, "listener", + createListenerFilterMatcher(proto_config)); ret.push_back(std::move(filter_config_provider)); } else { ENVOY_LOG(debug, " config: {}", diff --git a/test/common/filter/BUILD b/test/common/filter/BUILD index 1d711f510766..5c1c559183fe 100644 --- a/test/common/filter/BUILD +++ b/test/common/filter/BUILD @@ -25,7 +25,6 @@ envoy_cc_test( "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", diff --git a/test/common/filter/config_discovery_impl_test.cc b/test/common/filter/config_discovery_impl_test.cc index bb0512f867d4..ac645a2eefe0 100644 --- a/test/common/filter/config_discovery_impl_test.cc +++ b/test/common/filter/config_discovery_impl_test.cc @@ -24,7 +24,6 @@ #include "test/mocks/upstream/mocks.h" #include "test/test_common/printers.h" #include "test/test_common/simulated_time_system.h" -#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "absl/strings/substitute.h" @@ -104,7 +103,7 @@ class FilterConfigDiscoveryImplTest : public FilterConfigDiscoveryTestBase { } return filter_config_provider_manager_->createDynamicFilterConfigProvider( - config_source, name, factory_context_.getServerFactoryContext(), factory_context_, "xds.", + config_source, name, factory_context_.getServerFactoryContext(), factory_context_, last_filter_config, getFilterType(), getMatcher()); } @@ -281,32 +280,6 @@ TYPED_TEST(FilterConfigDiscoveryImplTestParameter, Basic) { } } -TYPED_TEST(FilterConfigDiscoveryImplTestParameter, BasicDeprecatedStatPrefix) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.top_level_ecds_stats", "false"}}); - - InSequence s; - TypeParam config_discovery_test; - config_discovery_test.setup(); - EXPECT_EQ("foo", config_discovery_test.provider_->name()); - EXPECT_EQ(absl::nullopt, config_discovery_test.provider_->config()); - - const auto response = config_discovery_test.createResponse("1", "foo"); - const auto decoded_resources = - TestUtility::decodeResources(response); - - EXPECT_CALL(config_discovery_test.init_watcher_, ready()); - config_discovery_test.callbacks_->onConfigUpdate(decoded_resources.refvec_, - response.version_info()); - EXPECT_NE(absl::nullopt, config_discovery_test.provider_->config()); - EXPECT_EQ(1UL, - config_discovery_test.scope_.counter("xds.extension_config_discovery.foo.config_reload") - .value()); - EXPECT_EQ(0UL, - config_discovery_test.scope_.counter("xds.extension_config_discovery.foo.config_fail") - .value()); -} - TYPED_TEST(FilterConfigDiscoveryImplTestParameter, ConfigFailed) { InSequence s; TypeParam config_discovery_test; @@ -459,8 +432,8 @@ TYPED_TEST(FilterConfigDiscoveryImplTestParameter, WrongDefaultConfig) { EXPECT_THROW_WITH_MESSAGE( config_discovery_test.filter_config_provider_manager_->createDynamicFilterConfigProvider( config_source, "foo", config_discovery_test.factory_context_.getServerFactoryContext(), - config_discovery_test.factory_context_, "xds.", true, - config_discovery_test.getFilterType(), config_discovery_test.getMatcher()), + config_discovery_test.factory_context_, true, config_discovery_test.getFilterType(), + config_discovery_test.getMatcher()), EnvoyException, "Error: cannot find filter factory foo for default filter " "configuration with type URL " @@ -496,9 +469,9 @@ TYPED_TEST(FilterConfigDiscoveryImplTestParameter, TerminalFilterInvalid) { EnvoyException, "Error: terminal filter named foo of type envoy.filters.http.router must be the last filter " "in a http filter chain."); - EXPECT_EQ(0UL, - config_discovery_test.scope_.counter("xds.extension_config_discovery.foo.config_reload") - .value()); + EXPECT_EQ( + 0UL, + config_discovery_test.scope_.counter("extension_config_discovery.foo.config_reload").value()); } // TCP listener filter matcher test. From ca9de5ae6e93193caedf5dece427e9d39f43ddd8 Mon Sep 17 00:00:00 2001 From: doujiang24 Date: Fri, 4 Nov 2022 17:00:00 +0800 Subject: [PATCH 105/112] improve the run_envoy_docker.sh compatibility on mac. (#23832) Since the stat on mac missing the -c option, i.e. stat: illegal option -- c usage: stat [-FLnq] [-f format | -l | -r | -s | -x] [-t timefmt] [file ...] Signed-off-by: doujiang24 --- ci/run_envoy_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index cad4b993cd2f..49eac0cb67fb 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -41,7 +41,7 @@ else BUILD_DIR_MOUNT_DEST=/build SOURCE_DIR="${PWD}" SOURCE_DIR_MOUNT_DEST=/source - DOCKER_GID="$(stat -c '%g' /var/run/docker.sock)" + DOCKER_GID="$(stat -c %g /var/run/docker.sock 2>/dev/null || stat -f %g /var/run/docker.sock)" START_COMMAND=("/bin/bash" "-lc" "groupadd --gid ${DOCKER_GID} -f envoygroup \ && useradd -o --uid $(id -u) --gid ${DOCKER_GID} --no-create-home --home-dir /build envoybuild \ && usermod -a -G pcap envoybuild \ From 51a76ca5b916d9e992ee5a4742f653a9ff88d926 Mon Sep 17 00:00:00 2001 From: Loong Dai Date: Fri, 4 Nov 2022 17:03:20 +0800 Subject: [PATCH 106/112] ci: accelerate local build (#23430) Signed-off-by: Loong --- ci/do_ci.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 034b35a6e644..e88b2a2ada7d 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -91,13 +91,15 @@ function cp_binary_for_image_build() { mkdir -p "${BASE_TARGET_DIR}"/"${TARGET_DIR}"_stripped strip "${FINAL_DELIVERY_DIR}"/envoy -o "${BASE_TARGET_DIR}"/"${TARGET_DIR}"_stripped/envoy - # Copy for azp which doesn't preserve permissions, creating a tar archive - tar czf "${ENVOY_BUILD_DIR}"/"${EXE_NAME}"_binary.tar.gz -C "${BASE_TARGET_DIR}" "${TARGET_DIR}" "${TARGET_DIR}"_stripped + # only if BUILD_REASON exists (running in AZP) + if [[ "${BUILD_REASON}" ]]; then + # Copy for azp which doesn't preserve permissions + tar czf "${ENVOY_BUILD_DIR}"/"${EXE_NAME}"_binary.tar.gz -C "${BASE_TARGET_DIR}" "${TARGET_DIR}" "${TARGET_DIR}"_stripped - # Remove binaries to save space, only if BUILD_REASON exists (running in AZP) - [[ -z "${BUILD_REASON}" ]] || \ + # Remove binaries to save space rm -rf "${BASE_TARGET_DIR:?}"/"${TARGET_DIR}" "${BASE_TARGET_DIR:?}"/"${TARGET_DIR}"_stripped "${FINAL_DELIVERY_DIR:?}"/envoy{,.dwp} \ bazel-bin/"${ENVOY_BIN}"{,.dwp} + fi } function bazel_binary_build() { From 4b7e144fa9edaa4cd904b94a91b496b3063afdc5 Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Fri, 4 Nov 2022 10:46:37 -0400 Subject: [PATCH 107/112] header-parser: fix config regex infinite loop (#23541) Signed-off-by: Adi Suissa-Peleg --- source/common/router/header_parser_utils.cc | 30 +++++++++-------- test/common/router/header_formatter_test.cc | 3 ++ .../header_parser_corpus/invalid_metadata | 8 +++++ .../router/route_corpus/regex_parsing_error | 32 +++++++++++++++++++ 4 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 test/common/router/header_parser_corpus/invalid_metadata create mode 100644 test/common/router/route_corpus/regex_parsing_error diff --git a/source/common/router/header_parser_utils.cc b/source/common/router/header_parser_utils.cc index b2ebeb12e3b0..39136cfce837 100644 --- a/source/common/router/header_parser_utils.cc +++ b/source/common/router/header_parser_utils.cc @@ -5,6 +5,7 @@ #include "source/common/json/json_loader.h" #include "source/common/router/header_parser.h" +#include "absl/strings/str_replace.h" #include "re2/re2.h" namespace Envoy { @@ -25,34 +26,34 @@ std::string HeaderParser::translateMetadataFormat(const std::string& header_valu const re2::RE2& re = getMetadataTranslatorPattern(); ASSERT(re.ok()); std::string new_header_value = header_value; - re2::StringPiece json_array, metadata_type; - while (re.PartialMatch(new_header_value, re, &metadata_type, &json_array)) { - std::string new_format; + re2::StringPiece matches[3]; + while (re.Match(new_header_value, 0, new_header_value.size(), re2::RE2::UNANCHORED, matches, 3)) { TRY_ASSERT_MAIN_THREAD { - Json::ObjectSharedPtr parsed_params = Json::Factory::loadFromString(json_array.as_string()); + std::string new_format; + Json::ObjectSharedPtr parsed_params = Json::Factory::loadFromString(std::string(matches[2])); // The given json string may be an invalid object or with an empty object array. if (parsed_params == nullptr || parsed_params->asObjectArray().empty()) { // return original value - return new_header_value; + return header_value; } new_format = parsed_params->asObjectArray()[0]->asString(); for (size_t i = 1; i < parsed_params->asObjectArray().size(); i++) { new_format += ":" + parsed_params->asObjectArray()[i]->asString(); } - new_format = "%" + metadata_type.as_string() + "_METADATA(" + new_format + ")%"; - + new_format = "%" + matches[1].as_string() + "_METADATA(" + new_format + ")%"; ENVOY_LOG_MISC( warn, "Header formatter: JSON format of {} parameters has been obsoleted. Use colon format: {}", - metadata_type.as_string() + "_METADATA", new_format.c_str()); + matches[1].as_string() + "_METADATA", new_format.c_str()); - re2::RE2::Replace(&new_header_value, re, new_format); + int subs = absl::StrReplaceAll({{matches[0].as_string(), new_format}}, &new_header_value); + ASSERT(subs > 0); } END_TRY catch (Json::Exception& e) { - return new_header_value; + return header_value; } } @@ -73,13 +74,14 @@ std::string HeaderParser::translatePerRequestState(const std::string& header_val const re2::RE2& re = getPerRequestTranslatorPattern(); ASSERT(re.ok()); std::string new_header_value = header_value; - re2::StringPiece required_state; - while (re.PartialMatch(new_header_value, re, &required_state)) { - std::string new_format = "%FILTER_STATE(" + required_state.as_string() + ":PLAIN)%"; + re2::StringPiece matches[2]; + while (re.Match(new_header_value, 0, new_header_value.size(), re2::RE2::UNANCHORED, matches, 2)) { + const std::string new_format = "%FILTER_STATE(" + matches[1].as_string() + ":PLAIN)%"; ENVOY_LOG_MISC(warn, "PER_REQUEST_STATE header formatter has been obsoleted. Use {}", new_format.c_str()); - re2::RE2::Replace(&new_header_value, re, new_format); + int subs = absl::StrReplaceAll({{matches[0].as_string(), new_format}}, &new_header_value); + ASSERT(subs > 0); } return new_header_value; } diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index 0a85bc804f73..86de6e661f10 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -1290,6 +1290,7 @@ TEST(HeaderParser, TestMetadataTranslatorExceptions) { static const std::string test_cases[] = { "%UPSTREAM_METADATA([\"a\" - \"b\"])%", "%UPSTREAM_METADATA(\t [ \t\t ] \t)%", + "%UPSTREAM_METADATA([\"udp{VTA(r%%%%%TA(r%%%%%b\\\\\\rin\\rsE(r%%%%%b\\\\\\rsi", }; for (const auto& test_case : test_cases) { EXPECT_EQ(test_case, HeaderParser::translateMetadataFormat(test_case)); @@ -1307,6 +1308,8 @@ TEST(HeaderParser, TestPerFilterStateTranslator) { "%FILTER_STATE(some-state:other-state:PLAIN)%"}, {"%PER_REQUEST_STATE(some-state)% %PER_REQUEST_STATE(other-state)%", "%FILTER_STATE(some-state:PLAIN)% %FILTER_STATE(other-state:PLAIN)%"}, + {"%PER_REQUEST_STATE(\\0)%", "%FILTER_STATE(\\0:PLAIN)%"}, + {"%PER_REQUEST_STATE(\\1)%", "%FILTER_STATE(\\1:PLAIN)%"}, }; for (const auto& test_case : test_cases) { diff --git a/test/common/router/header_parser_corpus/invalid_metadata b/test/common/router/header_parser_corpus/invalid_metadata new file mode 100644 index 000000000000..2d28ee13afe7 --- /dev/null +++ b/test/common/router/header_parser_corpus/invalid_metadata @@ -0,0 +1,8 @@ +headers_to_add { + header { + key: "d" + value: "%UPSTREAM_METADATA([\"udp{VTART_TIME(r%%%%%TART_TIME(r%%%%%b\\\\\\rsmonin\\rsionE(r%%%%%b\\\\\\rsioning.UPSTREAM_LOCA\\\\\\b\\\\\\r=i\\\\\\\\b\\\\\\r\\\\DOW\344\230\272REAM_LOCAL_SUBJECTR(\\\\\\\\\\\\_\\\\\\b\\\\\\rsioning.p\", \"TART_TIME(r%%[Z\362\235\243\243\303\204smonin\\rsionE(r%%%%%b\\\\\\rsioning.UPWTREAM_LOCAL_PORT\\\\RAILERh\\\\\\\\\\\\b\\\\\\r\\\\\\\\\\\\\\b\\\\\\rsioning.p\", \"TART_T\\\\b\\\\\\rsioning.p\", \"TART_TIME(r%%%%%b\\\\\\rsiEAM_LOCAL_SUBJECTR(\\\\\\\\\\\\_\\\\\\b\\\\\\rsioning.p\", \"TART_TIME(r%%%%%b\\\\\\rsmonin\\rsionE(r%%%%%b\\\\\\rsion%%%%b\\\\\\rsmonin\\rsionE(r%%%%%b\\\\\\rsi=i\\\\\\\\b\\\\\\r\\\\DOW\356\255\242REAM_r%%%%%b\\\\\\rsmonin\\rsionE(r%%%%%b\\\\\\rsioning.UPSrsionE(r%%%%%b\\\\\\rsion%%%%b\\\\\\rsmonin\\rsionE(r%%%%%b\\\\\\rsi=i\\\\\\\\b\\\\\\r\\\\DOW\356\255\242REAM_r%%%%%b\\\\\\rsmonin\\rsionE(r%%%%%b\\\\\\rsioning.UPSTREAM_LOCAL_PORT\\\\RAILERh\\\\\\\\\\\\b\\\\\\r\\\\\\\\\\\\\\b\\\\\\rsioning%\"])%%DOWNSTREAM_PEER_CERT_V_END%" + } + append { + } +} diff --git a/test/common/router/route_corpus/regex_parsing_error b/test/common/router/route_corpus/regex_parsing_error new file mode 100644 index 000000000000..50caaf4f85e4 --- /dev/null +++ b/test/common/router/route_corpus/regex_parsing_error @@ -0,0 +1,32 @@ +config { + virtual_hosts { + name: "." + domains: "*" + request_headers_to_add { + header { + key: "x-forwarded-proto" + value: "%START_TIME((%%%f%\034f%256\\002\\0N\\ss)%" + } + } + matcher { + on_no_match { + action { + name: "." + typed_config { + type_url: "m/envoy.config.route.v3.Route" + value: "\n\002\n\000\022\t\n\001v*\0015\242\002\000J\005\n\003\n\0011JF\nB\n\001$\022=%START_TIME((%%%fenvoy.filters.http.router%\034f%256\\002\\0N\\ss)% \001J\005\n\003\n\001$J\205\001\n\202\001\n\001$\022}%START_TIME((%%%fenvoy%PER_REQUEST_STATE(%fenvoy.type.v3.Int64Ra%TUEST_STATE(%f%ss[%%s.filters.http.router%\034f%256\\002\\0N\\ss)%J\010\n\006\n\0011\022\001\003b\001?b\021x-forwarded-protob\021x-forwarded-protor\001v\202\001\000" + } + } + } + } + } +} +headers { + headers { + key: "x-forwarded-proto" + value: "1" + } + headers { + key: ":path" + } +} From e21db7dc31d71298a7f84d7d581ed7d950afde8f Mon Sep 17 00:00:00 2001 From: Ryan Hamilton Date: Fri, 4 Nov 2022 07:47:24 -0700 Subject: [PATCH 108/112] deps: Update googleurl (#23824) Includes upstream fixes: https://quiche-review.googlesource.com/c/googleurl/+/11260 https://quiche-review.googlesource.com/c/googleurl/+/11220 https://quiche-review.googlesource.com/c/googleurl/+/11280 Signed-off-by: Ryan Hamilton rch@google.com Signed-off-by: Ryan Hamilton --- bazel/external/googleurl.patch | 24 ------------------------ bazel/repository_locations.bzl | 8 ++++---- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/bazel/external/googleurl.patch b/bazel/external/googleurl.patch index 442ec4e9290b..3cb364755316 100644 --- a/bazel/external/googleurl.patch +++ b/bazel/external/googleurl.patch @@ -116,27 +116,3 @@ index f881c04..4e4f7df 100644 // The annotation changed compiler output and increased binary size so disable // for official builds. // TODO(crbug.com/1320670): Remove when issue is resolved. - -# TODO(keith): Remove when https://quiche-review.googlesource.com/c/googleurl/+/11220 and https://quiche-review.googlesource.com/c/googleurl/+/11260 land - -diff --git a/base/BUILD b/base/BUILD -index 60ca578..c922b73 100644 ---- a/base/BUILD -+++ b/base/BUILD -@@ -40,6 +40,8 @@ cc_library( - "numerics/clamped_math.h", - "numerics/clamped_math_impl.h", - "numerics/safe_conversions.h", -+ "numerics/safe_conversions_arm_impl.h", -+ "numerics/safe_math_arm_impl.h", - "numerics/safe_conversions_impl.h", - "numerics/safe_math_clang_gcc_impl.h", - "numerics/safe_math_shared_impl.h", -@@ -55,6 +56,7 @@ cc_library( - "strings/string_number_conversions.h", - "strings/utf_string_conversions.h", - "strings/utf_string_conversion_utils.h", -+ "win/win_handle_types.h", - ] + build_config.strings_hdrs, - copts = build_config.default_copts, - textual_hdrs = [ diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 0a9e62b069d8..f0c2818d74bd 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1094,13 +1094,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Chrome URL parsing library", project_desc = "Chrome URL parsing library", project_url = "https://quiche.googlesource.com/googleurl", - # Static snapshot of https://quiche.googlesource.com/googleurl/+archive/fd287250b7f0d876478d88dd7641ba8a2e130bd2.tar.gz - version = "fd287250b7f0d876478d88dd7641ba8a2e130bd2", - sha256 = "053e6d8c80c7c4159012254de72ec17cc67a9945e709fbe9ac4925afcdb40884", + # Static snapshot of https://quiche.googlesource.com/googleurl/+archive/dd4080fec0b443296c0ed0036e1e776df8813aa7.tar.gz + version = "dd4080fec0b443296c0ed0036e1e776df8813aa7", + sha256 = "59f14d4fb373083b9dc8d389f16bbb817b5f936d1d436aa67e16eb6936028a51", urls = ["https://storage.googleapis.com/quiche-envoy-integration/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], extensions = [], - release_date = "2022-04-04", + release_date = "2022-11-03", cpe = "N/A", license = "googleurl", license_url = "https://quiche.googlesource.com/googleurl/+/{version}/LICENSE", From 1ff73200a2b6575eb60ebe05bcc324ce19eb835b Mon Sep 17 00:00:00 2001 From: zackzhangkai Date: Fri, 4 Nov 2022 22:50:56 +0800 Subject: [PATCH 109/112] fix doc link (#23829) Signed-off-by: zackzhangkai --- examples/route-mirror/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/route-mirror/README.md b/examples/route-mirror/README.md index 644009479424..899406a73f16 100644 --- a/examples/route-mirror/README.md +++ b/examples/route-mirror/README.md @@ -1,2 +1,2 @@ To learn about this sandbox and for instructions on how to run it please head over -to the [envoy docs](https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/route_mirror.html) +to the [envoy docs](https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/route-mirror.html) From 16b257347cfd2ed2a46506a5874cbfb11d916051 Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Fri, 4 Nov 2022 10:52:38 -0400 Subject: [PATCH 110/112] Fuzzing: Add corpus entry for file system filter (#23806) Signed-off-by: Yanjun Xiang --- ...zz-testcase-minimized-filter_fuzz_test-5022310482706432 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-5022310482706432 diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-5022310482706432 b/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-5022310482706432 new file mode 100644 index 000000000000..ce9def358697 --- /dev/null +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-5022310482706432 @@ -0,0 +1,7 @@ +config { + name: "envoy.filters.http.match_delegate" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher" + value: "\022w\n\001.\022r\ndtype.googleapis.com/envoy.extensions.filters.http.file_system_buffer.v3.FileSystemBufferFilterConfig\022\n\n\010\022\006\010\200\200\360\202\003" + } +} From 4a7c8bf2dddb656032d542be2844163449f294de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Nov 2022 11:08:04 -0400 Subject: [PATCH 111/112] build(deps): bump golang from `e4dcdac` to `8558ae6` in /examples/ext_authz/auth/grpc-service (#23805) build(deps): bump golang in /examples/ext_authz/auth/grpc-service Bumps golang from `e4dcdac` to `8558ae6`. Signed-off-by: dependabot[bot] --- examples/ext_authz/auth/grpc-service/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ext_authz/auth/grpc-service/Dockerfile b/examples/ext_authz/auth/grpc-service/Dockerfile index ea4381962dac..99d1ac592644 100644 --- a/examples/ext_authz/auth/grpc-service/Dockerfile +++ b/examples/ext_authz/auth/grpc-service/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:alpine@sha256:e4dcdac3ed37d8c2b3b8bcef2909573b2ad9c2ab53ba53c608909e8b89ccee36 AS builder +FROM golang:alpine@sha256:8558ae624304387d18694b9ea065cc9813dd4f7f9bd5073edb237541f2d0561b AS builder RUN apk --no-cache add make COPY . /app From 6eaaa255541844b11b34c23be557a58c5e95c1ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Nov 2022 11:10:06 -0400 Subject: [PATCH 112/112] build(deps): bump openpolicyagent/opa from 0.45.0-istio to 0.46.1-istio in /examples/ext_authz (#23822) build(deps): bump openpolicyagent/opa in /examples/ext_authz Bumps openpolicyagent/opa from 0.45.0-istio to 0.46.1-istio. Signed-off-by: dependabot[bot] --- examples/ext_authz/Dockerfile-opa | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ext_authz/Dockerfile-opa b/examples/ext_authz/Dockerfile-opa index 6842c4b65fa7..304ea7fbbd89 100644 --- a/examples/ext_authz/Dockerfile-opa +++ b/examples/ext_authz/Dockerfile-opa @@ -1 +1 @@ -FROM openpolicyagent/opa:0.45.0-istio@sha256:cfe64a704b37ac6a03e703390f149ffb398871bf23f09d34fed6bd49105122b8 +FROM openpolicyagent/opa:0.46.1-istio@sha256:c1f1c9f8dff56fbb18f44d4809ca6ee7ed1fd3fcacd6f06f15dc5d2c0d27a8b0