diff --git a/.bazelrc b/.bazelrc index c01fb9a2464c..637ac2a593e3 100644 --- a/.bazelrc +++ b/.bazelrc @@ -91,7 +91,7 @@ build:clang-pch --spawn_strategy=local build:clang-pch --define=ENVOY_CLANG_PCH=1 # Use gold linker for gcc compiler. -build:gcc --linkopt=-fuse-ld=gold +build:gcc --linkopt=-fuse-ld=gold --host_linkopt=-fuse-ld=gold build:gcc --test_env=HEAPCHECK= build:gcc --action_env=BAZEL_COMPILER=gcc build:gcc --action_env=CC=gcc --action_env=CXX=g++ diff --git a/.github/workflows/_check_coverage.yml b/.github/workflows/_check_coverage.yml index 1de9fecb0b5d..3ad92610f966 100644 --- a/.github/workflows/_check_coverage.yml +++ b/.github/workflows/_check_coverage.yml @@ -42,6 +42,12 @@ jobs: lower than limit rbe: true request: ${{ inputs.request }} + steps-post: | + - run: ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ matrix.target }}-upload' + shell: bash + env: + GCS_ARTIFACT_BUCKET: ${{ inputs.trusted && 'envoy-postsubmit' || 'envoy-pr' }} + GCS_REDIRECT_PATH: ${{ fromJSON(inputs.request).request.pr || fromJSON(inputs.request).request.target-branch }} target: ${{ matrix.target }} timeout-minutes: 180 trusted: ${{ inputs.trusted }} diff --git a/.github/workflows/_precheck_publish.yml b/.github/workflows/_precheck_publish.yml index 2c4f0c456abc..a8b6ae02a4ef 100644 --- a/.github/workflows/_precheck_publish.yml +++ b/.github/workflows/_precheck_publish.yml @@ -5,6 +5,9 @@ permissions: on: workflow_call: + secrets: + gcp-key: + required: true inputs: request: type: string @@ -20,6 +23,8 @@ concurrency: jobs: publish: + secrets: + gcp-key: ${{ secrets.gcp-key }} permissions: contents: read packages: read @@ -30,6 +35,7 @@ jobs: cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && '-arm64' || '' }} concurrency-suffix: -${{ matrix.target }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} + gcs-only: "true" rbe: ${{ matrix.rbe }} request: ${{ inputs.request }} runs-on: ${{ matrix.runs-on || 'ubuntu-24.04' }} @@ -38,6 +44,7 @@ jobs: ERROR error: Error: + steps-post: ${{ matrix.steps-post }} target: ${{ matrix.target }} target-suffix: ${{ matrix.target-suffix }} trusted: ${{ inputs.trusted }} @@ -67,3 +74,9 @@ jobs: --config=remote-envoy-engflow --config=docs-ci rbe: true + steps-post: | + - run: ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload' + shell: bash + env: + GCS_ARTIFACT_BUCKET: ${{ inputs.trusted && 'envoy-postsubmit' || 'envoy-pr' }} + GCS_REDIRECT_PATH: ${{ fromJSON(inputs.request).request.pr || fromJSON(inputs.request).request.target-branch }} diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml index 1c9766d19dc2..612b8f241001 100644 --- a/.github/workflows/_run.yml +++ b/.github/workflows/_run.yml @@ -69,6 +69,8 @@ on: Error: fail-match: type: string + gcs-only: + type: string import-gpg: type: boolean default: false @@ -277,9 +279,12 @@ jobs: GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -p "${{ runner.temp }}" -t gcp_service_account.XXXXXX.json) echo "${{ secrets.gcp-key }}" | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}" GCP_SERVICE_ACCOUNT_KEY_FILE="$(basename "${GCP_SERVICE_ACCOUNT_KEY_PATH}")" + echo "GCP_SERVICE_ACCOUNT_KEY_PATH=/build/${GCP_SERVICE_ACCOUNT_KEY_FILE}" >> "$GITHUB_ENV" + if [[ "${{ inputs.gcs-only }}" != "" ]]; then + exit 0 + fi BAZEL_BUILD_EXTRA_OPTIONS="--google_credentials=/build/${GCP_SERVICE_ACCOUNT_KEY_FILE} --config=remote-ci --config=rbe-google" echo "BAZEL_BUILD_EXTRA_OPTIONS=${BAZEL_BUILD_EXTRA_OPTIONS}" >> "$GITHUB_ENV" - echo "GCP_SERVICE_ACCOUNT_KEY_PATH=${GCP_SERVICE_ACCOUNT_KEY_PATH}" >> "$GITHUB_ENV" - uses: envoyproxy/toolshed/gh-actions/github/run@actions-v0.2.36 name: Run CI ${{ inputs.command }} ${{ inputs.target }} diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 5101ab9d19c1..8a3e2ad85ef0 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -34,7 +34,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@461ef6c76dfe95d5c364de2f431ddbd31a417628 # codeql-bundle-v3.26.9 + uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -64,6 +64,7 @@ jobs: --spawn_strategy=local \ --discard_analysis_cache \ --nouse_action_cache \ + --features="-layering_check" \ --config=clang-libc++ \ --config=ci \ //source/common/http/... @@ -73,4 +74,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@461ef6c76dfe95d5c364de2f431ddbd31a417628 # codeql-bundle-v3.26.9 + uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index ff5dad1275f0..f336d11cc80b 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -65,7 +65,7 @@ jobs: - name: Initialize CodeQL if: ${{ env.BUILD_TARGETS != '' }} - uses: github/codeql-action/init@461ef6c76dfe95d5c364de2f431ddbd31a417628 # codeql-bundle-v3.26.9 + uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 with: languages: cpp @@ -96,6 +96,7 @@ jobs: --spawn_strategy=local \ --discard_analysis_cache \ --nouse_action_cache \ + --features="-layering_check" \ --config=clang-libc++ \ --config=ci \ $BUILD_TARGETS @@ -108,4 +109,4 @@ jobs: - name: Perform CodeQL Analysis if: ${{ env.BUILD_TARGETS != '' }} - uses: github/codeql-action/analyze@461ef6c76dfe95d5c364de2f431ddbd31a417628 # codeql-bundle-v3.26.9 + uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # codeql-bundle-v3.26.11 diff --git a/.github/workflows/envoy-checks.yml b/.github/workflows/envoy-checks.yml index ee6f9d94d5b3..08422f5ad5c1 100644 --- a/.github/workflows/envoy-checks.yml +++ b/.github/workflows/envoy-checks.yml @@ -59,7 +59,7 @@ jobs: coverage: secrets: - gcp-key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + gcp-key: ${{ fromJSON(needs.load.outputs.trusted) && secrets.GCP_SERVICE_ACCOUNT_KEY_TRUSTED || secrets.GCP_SERVICE_ACCOUNT_KEY }} permissions: actions: read contents: read diff --git a/.github/workflows/envoy-prechecks.yml b/.github/workflows/envoy-prechecks.yml index 860856081368..35e9eed6fe2b 100644 --- a/.github/workflows/envoy-prechecks.yml +++ b/.github/workflows/envoy-prechecks.yml @@ -67,6 +67,8 @@ jobs: trusted: ${{ fromJSON(needs.load.outputs.trusted) }} publish: + secrets: + gcp-key: ${{ fromJSON(needs.load.outputs.trusted) && secrets.GCP_SERVICE_ACCOUNT_KEY_TRUSTED || secrets.GCP_SERVICE_ACCOUNT_KEY }} permissions: actions: read contents: read diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index f3075f3ebd56..88324734fb7e 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -40,6 +40,6 @@ jobs: retention-days: 5 - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@461ef6c76dfe95d5c364de2f431ddbd31a417628 # v3.26.9 + uses: github/codeql-action/upload-sarif@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11 with: sarif_file: results.sarif diff --git a/GOVERNANCE.md b/GOVERNANCE.md index ddadb310c63e..a09984c0850c 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -1,34 +1,22 @@ # Process for becoming a maintainer -## Your organization is not yet a maintainer +Becoming a maintainer generally means that you are going to be spending substantial time on +Envoy for the foreseeable future. You should have domain expertise and be extremely proficient in C++. -* Express interest to the senior maintainers that your organization is interested in becoming a - maintainer. Becoming a maintainer generally means that you are going to be spending substantial - time (>25%) on Envoy for the foreseeable future. You should have domain expertise and be extremely - proficient in C++. Ultimately your goal is to become a senior maintainer that will represent your - organization. +* Express interest to the + [envoy-maintainers](https://groups.google.com/forum/#!forum/envoy-announce) + that you are interested in becoming a maintainer and, if your company does not have pre-existing maintainers, + that your organization is interested in and willing to sponsoring a maintainer. * We will expect you to start contributing increasingly complicated PRs, under the guidance - of the existing senior maintainers. -* We may ask you to do some PRs from our backlog. + of the existing maintainers. +* We may ask you to fix some issues from our backlog. * As you gain experience with the code base and our standards, we will ask you to do code reviews for incoming PRs (i.e., all maintainers are expected to shoulder a proportional share of community reviews). -* After a period of approximately 2-3 months of working together and making sure we see eye to eye, - the existing senior maintainers will confer and decide whether to grant maintainer status or not. - We make no guarantees on the length of time this will take, but 2-3 months is the approximate - goal. - -## Your organization is currently a maintainer - -* First decide whether your organization really needs more people with maintainer access. Valid - reasons are "blast radius", a large organization that is working on multiple unrelated projects, - etc. -* Contact a senior maintainer for your organization and express interest. -* Start doing PRs and code reviews under the guidance of your senior maintainer. -* After a period of 1-2 months the existing senior maintainers will discuss granting "standard" - maintainer access. -* "Standard" maintainer access can be upgraded to "senior" maintainer access after another 1-2 - months of work and another conference of the existing senior committers. +* After a period of approximately 2-3 months of contributions demonstrating understanding of (at least parts of) + the Envoy code base, reach back out to the maintainers list asking for feedback. At this point, you will either + be granted maintainer status, or be given actionable feedback on any remaining gaps between the contributions + demonstrated and those expected of maintainers, at which point you can close those gaps and reach back out. ## Maintainer responsibilities diff --git a/api/envoy/config/core/v3/protocol.proto b/api/envoy/config/core/v3/protocol.proto index e566278600ab..423ec11b8cd9 100644 --- a/api/envoy/config/core/v3/protocol.proto +++ b/api/envoy/config/core/v3/protocol.proto @@ -262,6 +262,8 @@ message HttpProtocolOptions { // The maximum number of headers (request headers if configured on HttpConnectionManager, // response headers when configured on a cluster). // If unconfigured, the default maximum number of headers allowed is 100. + // The default value for requests can be overridden by setting runtime key ``envoy.reloadable_features.max_request_headers_count``. + // The default value for responses can be overridden by setting runtime key ``envoy.reloadable_features.max_response_headers_count``. // Downstream requests that exceed this limit will receive a 431 response for HTTP/1.x and cause a stream // reset for HTTP/2. // Upstream responses that exceed this limit will result in a 503 response. @@ -270,6 +272,7 @@ message HttpProtocolOptions { // The maximum size of response headers. // If unconfigured, the default is 60 KiB, except for HTTP/1 response headers which have a default // of 80KiB. + // The default value can be overridden by setting runtime key ``envoy.reloadable_features.max_response_headers_size_kb``. // Responses that exceed this limit will result in a 503 response. // In Envoy, this setting is only valid when configured on an upstream cluster, not on the // :ref:`HTTP Connection Manager diff --git a/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD b/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD index 1ef2f0c9bf47..ac9fd7c8abe8 100644 --- a/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD +++ b/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD @@ -7,6 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", + "//envoy/config/route/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/type/v3:pkg", "@com_github_cncf_xds//udpa/annotations:pkg", diff --git a/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto b/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto index a32475f352f3..82e38ed91d5a 100644 --- a/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto +++ b/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.local_ratelimit.v3; import "envoy/config/core/v3/base.proto"; +import "envoy/config/route/v3/route_components.proto"; import "envoy/extensions/common/ratelimit/v3/ratelimit.proto"; import "envoy/type/v3/http_status.proto"; import "envoy/type/v3/token_bucket.proto"; @@ -22,7 +23,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Local Rate limit :ref:`configuration overview `. // [#extension: envoy.filters.http.local_ratelimit] -// [#next-free-field: 17] +// [#next-free-field: 18] message LocalRateLimit { // The human readable prefix to use when emitting stats. string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; @@ -147,4 +148,23 @@ message LocalRateLimit { // of the default ``UNAVAILABLE`` gRPC code for a rate limited gRPC call. The // HTTP code will be 200 for a gRPC response. bool rate_limited_as_resource_exhausted = 15; + + // Rate limit configuration that is used to generate a list of descriptor entries based on + // the request context. The generated entries will be used to find one or multiple matched rate + // limit rule from the ``descriptors``. + // If this is set, then + // :ref:`VirtualHost.rate_limits` or + // :ref:`RouteAction.rate_limits` fields + // will be ignored. + // + // .. note:: + // Not all configuration fields of + // :ref:`rate limit config ` is supported at here. + // Following fields are not supported: + // + // 1. :ref:`rate limit stage `. + // 2. :ref:`dynamic metadata `. + // 3. :ref:`disable_key `. + // 4. :ref:`override limit `. + repeated config.route.v3.RateLimit rate_limits = 17; } diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 3d438ae87881..3b49f1329567 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -493,6 +493,7 @@ message HttpConnectionManager { // The maximum request headers size for incoming connections. // If unconfigured, the default max request headers allowed is 60 KiB. + // The default value can be overridden by setting runtime key ``envoy.reloadable_features.max_request_headers_size_kb``. // Requests that exceed this limit will receive a 431 response. // // Note: currently some protocol codecs impose limits on the maximum size of a single header: diff --git a/api/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.proto b/api/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.proto index a26a689e227c..2c9b5333f41d 100644 --- a/api/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.proto +++ b/api/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.proto @@ -5,7 +5,6 @@ package envoy.extensions.transport_sockets.http_11_proxy.v3; import "envoy/config/core/v3/base.proto"; import "udpa/annotations/status.proto"; -import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.transport_sockets.http_11_proxy.v3"; option java_outer_classname = "UpstreamHttp11ConnectProto"; @@ -34,6 +33,6 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // proxy address in ``config::core::v3::Address`` format. // message Http11ProxyUpstreamTransport { - // The underlying transport socket being wrapped. - config.core.v3.TransportSocket transport_socket = 1 [(validate.rules).message = {required: true}]; + // The underlying transport socket being wrapped. Defaults to plaintext (raw_buffer) if unset. + config.core.v3.TransportSocket transport_socket = 1; } diff --git a/api/envoy/extensions/transport_sockets/tls/v3/common.proto b/api/envoy/extensions/transport_sockets/tls/v3/common.proto index c1a3f5b33b34..46d86b65856e 100644 --- a/api/envoy/extensions/transport_sockets/tls/v3/common.proto +++ b/api/envoy/extensions/transport_sockets/tls/v3/common.proto @@ -290,12 +290,12 @@ message TlsSessionTicketKeys { // respect to the TLS handshake. // [#not-implemented-hide:] message CertificateProviderPluginInstance { - // Provider instance name. If not present, defaults to "default". + // Provider instance name. // // Instance names should generally be defined not in terms of the underlying provider // implementation (e.g., "file_watcher") but rather in terms of the function of the // certificates (e.g., "foo_deployment_identity"). - string instance_name = 1; + string instance_name = 1 [(validate.rules).string = {min_len: 1}]; // Opaque name used to specify certificate instances or types. For example, "ROOTCA" to specify // a root-certificate (validation context) or "example.com" to specify a certificate for a diff --git a/bazel/cel-cpp.patch b/bazel/cel-cpp.patch index 34dd41033e56..a2ce70b03c96 100644 --- a/bazel/cel-cpp.patch +++ b/bazel/cel-cpp.patch @@ -1,3 +1,15 @@ +diff --git a/base/attribute.h b/base/attribute.h +index 9462c180..d6dcce83 100644 +--- a/base/attribute.h ++++ b/base/attribute.h +@@ -23,6 +23,7 @@ + #include + + #include "absl/status/statusor.h" ++#include "absl/strings/str_cat.h" + #include "absl/strings/string_view.h" + #include "absl/types/optional.h" + #include "absl/types/span.h" diff --git a/base/memory.h b/base/memory.h index 3552e19..0fbe618 100644 --- a/base/memory.h diff --git a/bazel/external/boringssl_fips.genrule_cmd b/bazel/external/boringssl_fips.genrule_cmd index 51e6b72c7829..0f79c4e5fc3c 100755 --- a/bazel/external/boringssl_fips.genrule_cmd +++ b/bazel/external/boringssl_fips.genrule_cmd @@ -44,7 +44,7 @@ fi curl -sLO https://github.com/llvm/llvm-project/releases/download/llvmorg-"$VERSION"/clang+llvm-"$VERSION"-"$PLATFORM".tar.xz echo "$SHA256" clang+llvm-"$VERSION"-"$PLATFORM".tar.xz | sha256sum --check -tar xf clang+llvm-"$VERSION"-"$PLATFORM".tar.xz +tar xf clang+llvm-"$VERSION"-"$PLATFORM".tar.xz --no-same-owner printf "set(CMAKE_C_COMPILER \"clang\")\nset(CMAKE_CXX_COMPILER \"clang++\")\n" > ${HOME}/toolchain export PATH="$PWD/clang+llvm-$VERSION-$PLATFORM/bin:$PATH" @@ -66,7 +66,7 @@ fi curl -sLO https://dl.google.com/go/go"$VERSION"."$PLATFORM".tar.gz \ && echo "$SHA256" go"$VERSION"."$PLATFORM".tar.gz | sha256sum --check -tar xf go"$VERSION"."$PLATFORM".tar.gz +tar xf go"$VERSION"."$PLATFORM".tar.gz --no-same-owner export GOPATH="$PWD/gopath" export GOROOT="$PWD/go" @@ -82,7 +82,7 @@ VERSION=1.10.2 SHA256=ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed curl -sLO https://github.com/ninja-build/ninja/archive/refs/tags/v"$VERSION".tar.gz \ && echo "$SHA256" v"$VERSION".tar.gz | sha256sum --check -tar -xvf v"$VERSION".tar.gz +tar -xvf v"$VERSION".tar.gz --no-same-owner cd ninja-"$VERSION" python3 ./configure.py --bootstrap @@ -106,7 +106,7 @@ fi curl -sLO https://github.com/Kitware/CMake/releases/download/v"$VERSION"/cmake-"$VERSION"-"$PLATFORM".tar.gz \ && echo "$SHA256" cmake-"$VERSION"-"$PLATFORM".tar.gz | sha256sum --check -tar xf cmake-"$VERSION"-"$PLATFORM".tar.gz +tar xf cmake-"$VERSION"-"$PLATFORM".tar.gz --no-same-owner export PATH="$PWD/cmake-$VERSION-$PLATFORM/bin:$PATH" diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 3a334c4d4e44..09e998ef062c 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -4775,6 +4775,7 @@ envoy_quiche_platform_impl_cc_library( "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/log:flags", ], ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index b5233988c6e7..e42a6e7e2fd3 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -153,7 +153,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( # # !!! NOTE !!! # Anytime the FIPS BoringSSL version is upgraded, `bazel/external/boringssl_fips.genrule_cmd` must be updated to use the toolchain - # specified in the associated accredidation certificate, which can be found linked from + # specified in the associated accreditation certificate, which can be found linked from # https://boringssl.googlesource.com/boringssl/+/refs/heads/master/crypto/fipsmodule/FIPS.md, for example # https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/4735. version = "fips-20220613", @@ -181,12 +181,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Abseil", project_desc = "Open source collection of C++ libraries drawn from the most fundamental pieces of Google’s internal codebase", project_url = "https://abseil.io/", - version = "20230802.1", - sha256 = "987ce98f02eefbaf930d6e38ab16aa05737234d7afbab2d5c4ea7adbe50c28ed", + version = "20240722.0", + sha256 = "f50e5ac311a81382da7fa75b97310e4b9006474f9560ac46f54a9967f07d4ae3", strip_prefix = "abseil-cpp-{version}", urls = ["https://github.com/abseil/abseil-cpp/archive/{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2023-09-18", + release_date = "2024-08-01", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/abseil/abseil-cpp/blob/{version}/LICENSE", @@ -1208,12 +1208,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "171f6f89a6a119e8763f1216f8d85347f997cd3b", - sha256 = "3e0fec32dfa9c7568d4703516ee14c9e2316379e0a35f723d17a988be178e532", + version = "de8f411c1b387499c220ffd6702c43dd7ccaca00", + sha256 = "03a2855a70a3f22e0b723cc63f4d6b1817e894f35c2a441981c7f8152196713e", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2024-09-26", + release_date = "2024-10-07", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", @@ -1222,9 +1222,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Chrome URL parsing library", project_desc = "Chrome URL parsing library", project_url = "https://quiche.googlesource.com/googleurl", - # Static snapshot of https://quiche.googlesource.com/googleurl/+archive/dd4080fec0b443296c0ed0036e1e776df8813aa7.tar.gz version = "dd4080fec0b443296c0ed0036e1e776df8813aa7", - sha256 = "59f14d4fb373083b9dc8d389f16bbb817b5f936d1d436aa67e16eb6936028a51", + sha256 = "fc694942e8a7491dcc1dde1bddf48a31370a1f46fef862bc17acf07c34dc6325", + # Static snapshot of https://quiche.googlesource.com/googleurl/+archive/dd4080fec0b443296c0ed0036e1e776df8813aa7.tar.gz urls = ["https://storage.googleapis.com/quiche-envoy-integration/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], extensions = [], diff --git a/changelogs/current.yaml b/changelogs/current.yaml index b773fb76f626..2a24f8c539f6 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -105,6 +105,9 @@ minor_behavior_changes: Connect the QUIC UDP client connection sockets before use and sockets will only bind if the local address is specified. This behavior change can be reverted by setting the ``envoy_reloadable_features_quic_connect_client_udp_sockets`` runtime flag to ``false``. +- area: http_11_proxy + change: | + Make the inner ``transport_socket`` field optional in the proto configuration. - area: conn_handler change: | Enhanced listener filter chain execution to include the case that listener filter has maxReadBytes() of 0, @@ -112,10 +115,20 @@ minor_behavior_changes: - area: tracers change: | Set status code based on GRPC status code for OpenTelemetry tracers (previously unset). +- area: xds-failover + change: | + Add the ability to stick with either the primary or the failover xDS sources once Envoy connects to one of them. + This was added behind a runtime guard, to ensure that the move to the primary source can be properly validated, and + will be removed in the future. To allow sticksiyness the runtime flag + ``envoy.reloadable_features.xds_failover_to_primary_enabled`` must be explicitly set to ``false``. - area: http2 change: | Changes the default value of ``envoy.reloadable_features.http2_use_oghttp2`` to ``false``. This changes the codec used for HTTP/2 requests and responses to address to address stability concerns. This behavior can be reverted by setting the feature to ``true``. +- area: udp + change: | + Set Don't Fragment (DF) flag bit on IP packet header on UDP listener sockets and QUIC upstream connection sockets. This behavior + can be reverted by setting ``envoy.reloadable_features.udp_set_do_not_fragment`` to false. - area: access_log change: | Sanitize SNI for potential log injection. The invalid character will be replaced by ``_`` with an ``invalid:`` marker. If runtime @@ -306,7 +319,8 @@ new_features: - area: http change: | Added configuration setting for the :ref:`maximum size of response headers - ` in responses. + ` in responses. The default can + be overridden with runtime key ``envoy.reloadable_features.max_response_headers_size_kb``. - area: http_11_proxy change: | Added the option to configure the transport socket via locality or endpoint metadata. @@ -394,6 +408,9 @@ new_features: to configure the number of retries. If this field is not provided, the ``getaddrinfo`` resolver will retry indefinitely until it succeeds or the DNS query times out. This behavior can be reverted by setting the runtime guard ``envoy.reloadable_features.getaddrinfo_num_retries`` to false. +- area: getaddrinfo + change: | + Added ``envoy.enable_dfp_dns_trace`` runtime flag to enable DNS query trace in the DNS resolution details. - area: geoip change: | Added ``envoy.reloadable_features.mmdb_files_reload_enabled`` runtime flag that enables reload of mmdb files by default. @@ -431,6 +448,14 @@ new_features: change: | Added two new methods ``oidsPeerCertificate()`` and ``oidsLocalCertificate()`` to SSL connection object API :ref:`SSL connection info object `. +- area: local_ratelimit + change: | + Add the :ref:`rate_limits + ` + field to generate rate limit descriptors. If this field is set, the + :ref:`VirtualHost.rate_limits` or + :ref:`RouteAction.rate_limits` fields + will be ignored. - area: basic_auth change: | Added support to provide an override diff --git a/ci/do_ci.sh b/ci/do_ci.sh index f67bbca4fdee..99f29f661e11 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -57,7 +57,6 @@ FETCH_PROTO_TARGETS=( @com_github_bufbuild_buf//:bin/buf //tools/proto_format/...) -GCS_REDIRECT_PATH="${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-${BUILD_SOURCEBRANCHNAME}}" retry () { local n wait iterations diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index dcb86af9db9a..2d60de2f08a4 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -158,15 +158,14 @@ docker run --rm \ -e ENVOY_REPO \ -e ENVOY_TARBALL_DIR \ -e ENVOY_GEN_COMPDB_OPTIONS \ - -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ -e GCS_ARTIFACT_BUCKET \ + -e GCS_REDIRECT_PATH \ -e GITHUB_REF_NAME \ -e GITHUB_REF_TYPE \ -e GITHUB_TOKEN \ -e GITHUB_APP_ID \ -e GITHUB_INSTALL_ID \ -e MOBILE_DOCS_CHECKOUT_DIR \ - -e BUILD_SOURCEBRANCHNAME \ -e BAZELISK_BASE_URL \ -e ENVOY_BUILD_ARCH \ -e SYSTEM_STAGEDISPLAYNAME \ diff --git a/envoy/common/BUILD b/envoy/common/BUILD index 12c71b554bbf..c11152a6391c 100644 --- a/envoy/common/BUILD +++ b/envoy/common/BUILD @@ -127,6 +127,8 @@ envoy_cc_library( deps = [ ":pure_lib", ":scope_tracker_interface", + "//source/common/common:cleanup_lib", + "//source/common/common:macros", ], ) diff --git a/envoy/common/exception.h b/envoy/common/exception.h index 5734df1f6628..7b74e2094b39 100644 --- a/envoy/common/exception.h +++ b/envoy/common/exception.h @@ -30,8 +30,8 @@ class EnvoyException : public std::runtime_error { }; #define SET_AND_RETURN_IF_NOT_OK(check_status, set_status) \ - if (!check_status.ok()) { \ - set_status = check_status; \ + if (const absl::Status temp_status = check_status; !temp_status.ok()) { \ + set_status = temp_status; \ return; \ } diff --git a/envoy/common/execution_context.h b/envoy/common/execution_context.h index b723a221bcae..3580f8f2bd3b 100644 --- a/envoy/common/execution_context.h +++ b/envoy/common/execution_context.h @@ -6,6 +6,8 @@ #include "envoy/common/scope_tracker.h" #include "envoy/stream_info/stream_info.h" +#include "source/common/common/cleanup.h" +#include "source/common/common/macros.h" #include "source/common/common/non_copyable.h" namespace Envoy { @@ -15,6 +17,14 @@ namespace Envoy { static constexpr absl::string_view kConnectionExecutionContextFilterStateName = "envoy.network.connection_execution_context"; +namespace Http { +struct FilterContext; +} + +namespace Tracing { +class Span; +} + class ScopedExecutionContext; // ExecutionContext can be inherited by subclasses to represent arbitrary information associated @@ -22,6 +32,27 @@ class ScopedExecutionContext; // starts/ends. For an example usage, please see // https://github.com/envoyproxy/envoy/issues/32012. class ExecutionContext : public StreamInfo::FilterState::Object, NonCopyable { +public: + static void setEnabled(bool value) { enabled().store(value, std::memory_order_relaxed); } + + static bool isEnabled() { return enabled().load(std::memory_order_relaxed); } + + static ExecutionContext* fromStreamInfo(OptRef info) { + if (!isEnabled() || !info.has_value()) { + return nullptr; + } + const auto* const_context = info->filterState().getDataReadOnly( + kConnectionExecutionContextFilterStateName); + return const_cast(const_context); + } + + // Called when enters a scope in which |*span| is active. + // Returns an object that can do some cleanup when exits the scope. + virtual Envoy::Cleanup onScopeEnter(Envoy::Tracing::Span* span) PURE; + // Called when enters a scope in which |*filter_context| is active. + // Returns an object that can do some cleanup when exits the scope. + virtual Envoy::Cleanup onScopeEnter(const Http::FilterContext* filter_context) PURE; + protected: // Called when the current thread starts to run code on behalf of the owner of this object. // protected because it should only be called by ScopedExecutionContext. @@ -30,6 +61,9 @@ class ExecutionContext : public StreamInfo::FilterState::Object, NonCopyable { // protected because it should only be called by ScopedExecutionContext. virtual void deactivate() PURE; +private: + static std::atomic& enabled() { MUTABLE_CONSTRUCT_ON_FIRST_USE(std::atomic); } + friend class ScopedExecutionContext; }; @@ -47,7 +81,8 @@ class ScopedExecutionContext : NonCopyable { public: ScopedExecutionContext() : ScopedExecutionContext(nullptr) {} ScopedExecutionContext(const ScopeTrackedObject* object) - : context_(object != nullptr ? getExecutionContext(object->trackedStream()) : nullptr) { + : context_(object != nullptr ? ExecutionContext::fromStreamInfo(object->trackedStream()) + : nullptr) { if (context_ != nullptr) { context_->activate(); } @@ -66,18 +101,25 @@ class ScopedExecutionContext : NonCopyable { bool isNull() const { return context_ == nullptr; } private: - ExecutionContext* getExecutionContext(OptRef info) { - if (!info.has_value()) { - return nullptr; - } - const auto* const_context = info->filterState().getDataReadOnly( - kConnectionExecutionContextFilterStateName); - return const_cast(const_context); - } - ExecutionContext* context_; }; +#define ENVOY_EXECUTION_SCOPE_CAT_(a, b) a##b +#define ENVOY_EXECUTION_SCOPE_CAT(a, b) ENVOY_EXECUTION_SCOPE_CAT_(a, b) +// Invoked when |scopedObject| is active from the current line to the end of the current c++ scope. +// |trackedStream| is a OptRef from which a ExecutionContext is extracted. +// |scopedObject| is a pointer to a Envoy::Tracing::Span or a Http::FilterContext. +#define ENVOY_EXECUTION_SCOPE(trackedStream, scopedObject) \ + Envoy::Cleanup ENVOY_EXECUTION_SCOPE_CAT(on_scope_exit_, __LINE__) = \ + [execution_context = ExecutionContext::fromStreamInfo(trackedStream), \ + scoped_object = (scopedObject)] { \ + if (execution_context == nullptr) { \ + return Envoy::Cleanup::Noop(); \ + } \ + return execution_context->onScopeEnter(scoped_object); \ + }() +#else +#define ENVOY_EXECUTION_SCOPE(trackedStream, scopedObject) #endif } // namespace Envoy diff --git a/envoy/config/grpc_mux.h b/envoy/config/grpc_mux.h index a18c87bc193e..096c439b0bd0 100644 --- a/envoy/config/grpc_mux.h +++ b/envoy/config/grpc_mux.h @@ -2,10 +2,13 @@ #include +#include "envoy/common/backoff_strategy.h" #include "envoy/common/exception.h" #include "envoy/common/pure.h" +#include "envoy/config/custom_config_validators.h" #include "envoy/config/eds_resources_cache.h" #include "envoy/config/subscription.h" +#include "envoy/grpc/async_client.h" #include "envoy/stats/stats_macros.h" #include "source/common/common/cleanup.h" @@ -112,6 +115,16 @@ class GrpcMux { * @return EdsResourcesCacheOptRef optional eds resources cache for the gRPC-mux. */ virtual EdsResourcesCacheOptRef edsResourcesCache() PURE; + + /** + * Updates the current gRPC-Mux object to use a new gRPC client, and config. + */ + virtual absl::Status + updateMuxSource(Grpc::RawAsyncClientPtr&& primary_async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + CustomConfigValidatorsPtr&& custom_config_validators, Stats::Scope& scope, + BackOffStrategyPtr&& backoff_strategy, + const envoy::config::core::v3::ApiConfigSource& ads_config_source) PURE; }; using GrpcMuxPtr = std::unique_ptr; diff --git a/envoy/config/subscription.h b/envoy/config/subscription.h index 4bcd3e56b4e5..6ca9dc2403b5 100644 --- a/envoy/config/subscription.h +++ b/envoy/config/subscription.h @@ -191,10 +191,10 @@ class UntypedConfigUpdateCallbacks { * being updated. Accepted changes have their version_info reflected in subsequent * requests. */ - virtual void onConfigUpdate( - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) PURE; + virtual void + onConfigUpdate(absl::Span added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) PURE; /** * Called when either the Subscription is unable to fetch a config update or when onConfigUpdate diff --git a/envoy/config/xds_config_tracker.h b/envoy/config/xds_config_tracker.h index e1bbac0e0edc..5f0b3ddd740a 100644 --- a/envoy/config/xds_config_tracker.h +++ b/envoy/config/xds_config_tracker.h @@ -60,10 +60,10 @@ class XdsConfigTracker { * @param added_resources A list of decoded resources to add to the current state. * @param removed_resources A list of resources to remove from the current state. */ - virtual void onConfigAccepted( - const absl::string_view type_url, - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources) PURE; + virtual void + onConfigAccepted(const absl::string_view type_url, + absl::Span added_resources, + const Protobuf::RepeatedPtrField& removed_resources) PURE; /** * Invoked when xds configs are rejected during xDS ingestion. diff --git a/envoy/http/codec.h b/envoy/http/codec.h index 7408d4085a30..0297a8a32356 100644 --- a/envoy/http/codec.h +++ b/envoy/http/codec.h @@ -46,6 +46,8 @@ const char MaxResponseHeadersCountOverrideKey[] = "envoy.reloadable_features.max_response_headers_count"; const char MaxRequestHeadersSizeOverrideKey[] = "envoy.reloadable_features.max_request_headers_size_kb"; +const char MaxResponseHeadersSizeOverrideKey[] = + "envoy.reloadable_features.max_response_headers_size_kb"; class Stream; class RequestDecoder; diff --git a/envoy/network/BUILD b/envoy/network/BUILD index 95ab4e4677e5..160559ab8ad8 100644 --- a/envoy/network/BUILD +++ b/envoy/network/BUILD @@ -85,7 +85,10 @@ envoy_cc_library( envoy_cc_library( name = "dns_interface", hdrs = ["dns.h"], - deps = ["//envoy/network:address_interface"], + deps = [ + "//envoy/common:time_interface", + "//envoy/network:address_interface", + ], ) envoy_cc_library( diff --git a/envoy/network/dns.h b/envoy/network/dns.h index c7847babcc83..3b5fcf893c49 100644 --- a/envoy/network/dns.h +++ b/envoy/network/dns.h @@ -7,8 +7,10 @@ #include #include "envoy/common/pure.h" +#include "envoy/common/time.h" #include "envoy/network/address.h" +#include "absl/types/optional.h" #include "absl/types/variant.h" namespace Envoy { @@ -30,11 +32,31 @@ class ActiveDnsQuery { Timeout }; + /** Store the trace information. */ + struct Trace { + /** + * An identifier to store the trace information. The trace is `uint8_t` because the value can + * vary depending on the DNS resolver implementation. + */ + uint8_t trace_; + /** Store the current time of this trace. */ + MonotonicTime time_; + }; + /** * Cancel an outstanding DNS request. * @param reason supplies the cancel reason. */ virtual void cancel(CancelReason reason) PURE; + + /** + * Add a trace for the DNS query. The trace lifetime is tied to the lifetime of `ActiveQuery` and + * `ActiveQuery` will be destroyed upon query completion or cancellation. + */ + virtual void addTrace(uint8_t trace) PURE; + + /** Return the DNS query traces. */ + virtual std::string getTraces() PURE; }; /** diff --git a/envoy/network/socket.h b/envoy/network/socket.h index 39c492754964..bed7b7be35aa 100644 --- a/envoy/network/socket.h +++ b/envoy/network/socket.h @@ -175,6 +175,25 @@ static_assert(IP_RECVDSTADDR == IP_SENDSRCADDR); #define ENVOY_ATTACH_REUSEPORT_CBPF Network::SocketOptionName() #endif +#if !defined(ANDROID) && defined(__APPLE__) +// Only include TargetConditionals after testing ANDROID as some Android builds +// on the Mac have this header available and it's not needed unless the target +// is really an Apple platform. +#include +#if !defined(TARGET_OS_IPHONE) || !TARGET_OS_IPHONE +// MAC_OS +#define ENVOY_IP_DONTFRAG ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_DONTFRAG) +#define ENVOY_IPV6_DONTFRAG ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_DONTFRAG) +#endif +#endif + +#if !defined(ENVOY_IP_DONTFRAG) && defined(IP_PMTUDISC_DO) +#define ENVOY_IP_MTU_DISCOVER ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_MTU_DISCOVER) +#define ENVOY_IP_MTU_DISCOVER_VALUE IP_PMTUDISC_DO +#define ENVOY_IPV6_MTU_DISCOVER ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_MTU_DISCOVER) +#define ENVOY_IPV6_MTU_DISCOVER_VALUE IPV6_PMTUDISC_DO +#endif + /** * Interface representing a single filter chain info. */ diff --git a/envoy/stream_info/filter_state.h b/envoy/stream_info/filter_state.h index 7e7cfd7fdef0..f43ffe00eb3b 100644 --- a/envoy/stream_info/filter_state.h +++ b/envoy/stream_info/filter_state.h @@ -87,6 +87,8 @@ class FilterState { class Object { public: + using FieldType = absl::variant; + virtual ~Object() = default; /** @@ -102,21 +104,17 @@ class FilterState { * This method can be used to get an unstructured serialization result. */ virtual absl::optional serializeAsString() const { return absl::nullopt; } - }; - - /** - * Generic reflection support for the filter state objects. - */ - class ObjectReflection { - public: - virtual ~ObjectReflection() = default; - using FieldType = absl::variant; + /** + * @return bool true if the object supports field access. False if the object does not support + * field access. Default implementation returns false. + */ + virtual bool hasFieldSupport() const { return false; } /** - * @return FieldType a field value for a field name. + * @return FieldType a single state property or field value for a name. */ - virtual FieldType getField(absl::string_view) const PURE; + virtual FieldType getField(absl::string_view) const { return absl::monostate{}; } }; /** @@ -134,12 +132,6 @@ class FilterState { * is malformed. */ virtual std::unique_ptr createFromBytes(absl::string_view data) const PURE; - - /** - * @return std::unique_ptr for the runtime object - * Note that the reflection object is a view and should not outlive the object. - */ - virtual std::unique_ptr reflect(const Object*) const { return nullptr; } }; struct FilterObject { diff --git a/envoy/stream_info/stream_info.h b/envoy/stream_info/stream_info.h index 39ffd18cef6a..2b14446c28ba 100644 --- a/envoy/stream_info/stream_info.h +++ b/envoy/stream_info/stream_info.h @@ -354,8 +354,7 @@ struct UpstreamTiming { absl::optional upstream_handshake_complete_; }; -class DownstreamTiming { -public: +struct DownstreamTiming { void setValue(absl::string_view key, MonotonicTime value) { timings_[key] = value; } absl::optional getValue(absl::string_view value) const { @@ -410,7 +409,6 @@ class DownstreamTiming { last_downstream_header_rx_byte_received_ = time_source.monotonicTime(); } -private: absl::flat_hash_map timings_; // The time when the last byte of the request was received. absl::optional last_downstream_rx_byte_received_; diff --git a/mobile/library/cc/engine.cc b/mobile/library/cc/engine.cc index d94ce7480fe8..b4aa819cb121 100644 --- a/mobile/library/cc/engine.cc +++ b/mobile/library/cc/engine.cc @@ -1,5 +1,6 @@ #include "engine.h" +#include "library/common/engine_types.h" #include "library/common/internal_engine.h" #include "library/common/types/c_types.h" @@ -26,5 +27,9 @@ std::string Engine::dumpStats() { return engine_->dumpStats(); } envoy_status_t Engine::terminate() { return engine_->terminate(); } +void Engine::onDefaultNetworkChanged(NetworkType network) { + engine_->onDefaultNetworkChanged(network); +} + } // namespace Platform } // namespace Envoy diff --git a/mobile/library/cc/engine.h b/mobile/library/cc/engine.h index 172985a08a34..12302e178300 100644 --- a/mobile/library/cc/engine.h +++ b/mobile/library/cc/engine.h @@ -3,6 +3,7 @@ #include #include "library/cc/stream_client.h" +#include "library/common/engine_types.h" #include "library/common/types/c_types.h" namespace Envoy { @@ -20,6 +21,7 @@ class Engine : public std::enable_shared_from_this { std::string dumpStats(); StreamClientSharedPtr streamClient(); + void onDefaultNetworkChanged(NetworkType network); envoy_status_t terminate(); Envoy::InternalEngine* engine() { return engine_; } diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc index 53518b8cd984..26fafc8188b7 100644 --- a/mobile/library/cc/engine_builder.cc +++ b/mobile/library/cc/engine_builder.cc @@ -222,8 +222,8 @@ EngineBuilder& EngineBuilder::addQuicCanonicalSuffix(std::string suffix) { return *this; } -EngineBuilder& EngineBuilder::enablePortMigration(bool enable_port_migration) { - enable_port_migration_ = enable_port_migration; +EngineBuilder& EngineBuilder::setNumTimeoutsToTriggerPortMigration(int num_timeouts) { + num_timeouts_to_trigger_port_migration_ = num_timeouts; return *this; } @@ -267,6 +267,12 @@ EngineBuilder& EngineBuilder::setUpstreamTlsSni(std::string sni) { return *this; } +EngineBuilder& +EngineBuilder::setQuicConnectionIdleTimeoutSeconds(int quic_connection_idle_timeout_seconds) { + quic_connection_idle_timeout_seconds_ = quic_connection_idle_timeout_seconds; + return *this; +} + EngineBuilder& EngineBuilder::enablePlatformCertificatesValidation(bool platform_certificates_validation_on) { platform_certificates_validation_on_ = platform_certificates_validation_on; @@ -727,19 +733,19 @@ std::unique_ptr EngineBuilder::generate ->add_canonical_suffixes(suffix); } - if (enable_port_migration_) { + if (num_timeouts_to_trigger_port_migration_ > 0) { alpn_options.mutable_auto_config() ->mutable_http3_protocol_options() ->mutable_quic_protocol_options() ->mutable_num_timeouts_to_trigger_port_migration() - ->set_value(4); + ->set_value(num_timeouts_to_trigger_port_migration_); } alpn_options.mutable_auto_config() ->mutable_http3_protocol_options() ->mutable_quic_protocol_options() ->mutable_idle_network_timeout() - ->set_seconds(30); + ->set_seconds(quic_connection_idle_timeout_seconds_); base_cluster->mutable_transport_socket()->mutable_typed_config()->PackFrom(h3_proxy_socket); (*base_cluster->mutable_typed_extension_protocol_options()) @@ -845,6 +851,7 @@ std::unique_ptr EngineBuilder::generate } (*runtime_values.mutable_fields())["disallow_global_stats"].set_bool_value(true); + (*runtime_values.mutable_fields())["enable_dfp_dns_trace"].set_bool_value(true); ProtobufWkt::Struct& overload_values = *(*envoy_layer.mutable_fields())["overload"].mutable_struct_value(); (*overload_values.mutable_fields())["global_downstream_max_connections"].set_string_value( diff --git a/mobile/library/cc/engine_builder.h b/mobile/library/cc/engine_builder.h index 7636eab146a5..0decc13dd788 100644 --- a/mobile/library/cc/engine_builder.h +++ b/mobile/library/cc/engine_builder.h @@ -62,7 +62,8 @@ class EngineBuilder { EngineBuilder& setHttp3ClientConnectionOptions(std::string options); EngineBuilder& addQuicHint(std::string host, int port); EngineBuilder& addQuicCanonicalSuffix(std::string suffix); - EngineBuilder& enablePortMigration(bool enable_port_migration); + // 0 means port migration is disabled. + EngineBuilder& setNumTimeoutsToTriggerPortMigration(int num_timeouts); EngineBuilder& enableInterfaceBinding(bool interface_binding_on); EngineBuilder& enableDrainPostDnsRefresh(bool drain_post_dns_refresh_on); // Sets whether to use GRO for upstream UDP sockets (QUIC/HTTP3). @@ -104,6 +105,9 @@ class EngineBuilder { // outside of this range will be ignored. EngineBuilder& setNetworkThreadPriority(int thread_priority); + // Sets the QUIC connection idle timeout in seconds. + EngineBuilder& setQuicConnectionIdleTimeoutSeconds(int quic_connection_idle_timeout_seconds); + #if defined(__APPLE__) // Right now, this API is only used by Apple (iOS) to register the Apple proxy resolver API for // use in reading and using the system proxy settings. @@ -176,7 +180,7 @@ class EngineBuilder { std::string http3_client_connection_options_ = ""; std::vector> quic_hints_; std::vector quic_suffixes_; - bool enable_port_migration_ = false; + int num_timeouts_to_trigger_port_migration_ = 0; bool always_use_v6_ = false; #if defined(__APPLE__) // TODO(abeyad): once stable, consider setting the default to true. @@ -201,6 +205,8 @@ class EngineBuilder { // https://source.chromium.org/chromium/chromium/src/+/main:net/quic/quic_session_pool.cc;l=790-793;drc=7f04a8e033c23dede6beae129cd212e6d4473d72 // https://source.chromium.org/chromium/chromium/src/+/main:net/third_party/quiche/src/quiche/quic/core/quic_constants.h;l=43-47;drc=34ad7f3844f882baf3d31a6bc6e300acaa0e3fc8 int32_t udp_socket_send_buffer_size_ = 1452 * 20; + + int quic_connection_idle_timeout_seconds_ = 30; }; using EngineBuilderSharedPtr = std::shared_ptr; diff --git a/mobile/library/common/http/client.cc b/mobile/library/common/http/client.cc index 3a3a8f039e20..f1613aed8864 100644 --- a/mobile/library/common/http/client.cc +++ b/mobile/library/common/http/client.cc @@ -656,9 +656,9 @@ void Client::sendData(envoy_stream_t stream, Buffer::InstancePtr buffer, bool en direct_stream->wants_write_notification_ = false; // A new callback must be scheduled each time to capture any changes to the // DirectStream's callbacks from call to call. - scheduled_callback_ = dispatcher_.createSchedulableCallback( + direct_stream->scheduled_callback_ = dispatcher_.createSchedulableCallback( [direct_stream] { direct_stream->callbacks_->onSendWindowAvailable(); }); - scheduled_callback_->scheduleCallbackNextIteration(); + direct_stream->scheduled_callback_->scheduleCallbackNextIteration(); } else { // Otherwise, make sure the stack will send a notification when the // buffers are drained. @@ -699,7 +699,6 @@ void Client::cancelStream(envoy_stream_t stream) { // whether it was closed or not. Client::DirectStreamSharedPtr direct_stream = getStream(stream, GetStreamFilters::AllowForAllStreams); - scheduled_callback_ = nullptr; if (direct_stream) { // Attempt to latch the latest stream info. This will be a no-op if the stream // is already complete. @@ -759,6 +758,7 @@ void Client::removeStream(envoy_stream_t stream_handle) { "[S{}] removeStream is a private method that is only called with stream ids that exist", stream_handle)); + direct_stream->scheduled_callback_ = nullptr; // The DirectStream should live through synchronous code that already has a reference to it. // Hence why it is scheduled for deferred deletion. If this was all that was needed then it // would be sufficient to return a shared_ptr in getStream. However, deferred deletion is still diff --git a/mobile/library/common/http/client.h b/mobile/library/common/http/client.h index 046588e53fea..6c425747c97c 100644 --- a/mobile/library/common/http/client.h +++ b/mobile/library/common/http/client.h @@ -331,6 +331,7 @@ class Client : public Logger::Loggable { // Set true in explicit flow control mode if the library has sent body data and may want to // send more when buffer is available. bool wants_write_notification_{}; + Event::SchedulableCallbackPtr scheduled_callback_; // True if the bridge should operate in explicit flow control mode. // // In this mode only one callback can be sent to the bridge until more is @@ -383,7 +384,6 @@ class Client : public Logger::Loggable { ApiListenerPtr api_listener_; Event::ProvisionalDispatcher& dispatcher_; - Event::SchedulableCallbackPtr scheduled_callback_; HttpClientStats stats_; // The set of open streams, which can safely have request data sent on them // or response data received. diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java index 40f6f4dfc080..0cee67c9953c 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java @@ -46,7 +46,7 @@ public enum TrustChainVerification { public final List quicCanonicalSuffixes; public final boolean enableGzipDecompression; public final boolean enableBrotliDecompression; - public final boolean enablePortMigration; + public final int numTimeoutsToTriggerPortMigration; public final boolean enableSocketTagging; public final boolean enableInterfaceBinding; public final int h2ConnectionKeepaliveIdleIntervalMilliseconds; @@ -106,7 +106,8 @@ public enum TrustChainVerification { * decompression. * @param enableBrotliDecompression whether to enable response brotli * decompression. - * @param enablePortMigration whether to enable quic port migration. + * @param numTimeoutsToTriggerPortMigration number of timeouts to trigger port + * migration. * @param enableSocketTagging whether to enable socket tagging. * @param enableInterfaceBinding whether to allow interface binding. * @param h2ConnectionKeepaliveIdleIntervalMilliseconds rate in milliseconds seconds to send h2 @@ -139,11 +140,11 @@ public EnvoyConfiguration( boolean forceV6, boolean useGro, String http3ConnectionOptions, String http3ClientConnectionOptions, Map quicHints, List quicCanonicalSuffixes, boolean enableGzipDecompression, - boolean enableBrotliDecompression, boolean enablePortMigration, boolean enableSocketTagging, - boolean enableInterfaceBinding, int h2ConnectionKeepaliveIdleIntervalMilliseconds, - int h2ConnectionKeepaliveTimeoutSeconds, int maxConnectionsPerHost, - int streamIdleTimeoutSeconds, int perTryIdleTimeoutSeconds, String appVersion, String appId, - TrustChainVerification trustChainVerification, + boolean enableBrotliDecompression, int numTimeoutsToTriggerPortMigration, + boolean enableSocketTagging, boolean enableInterfaceBinding, + int h2ConnectionKeepaliveIdleIntervalMilliseconds, int h2ConnectionKeepaliveTimeoutSeconds, + int maxConnectionsPerHost, int streamIdleTimeoutSeconds, int perTryIdleTimeoutSeconds, + String appVersion, String appId, TrustChainVerification trustChainVerification, List nativeFilterChain, List httpPlatformFilterFactories, Map stringAccessors, @@ -180,7 +181,7 @@ public EnvoyConfiguration( this.quicCanonicalSuffixes = quicCanonicalSuffixes; this.enableGzipDecompression = enableGzipDecompression; this.enableBrotliDecompression = enableBrotliDecompression; - this.enablePortMigration = enablePortMigration; + this.numTimeoutsToTriggerPortMigration = numTimeoutsToTriggerPortMigration; this.enableSocketTagging = enableSocketTagging; this.enableInterfaceBinding = enableInterfaceBinding; this.h2ConnectionKeepaliveIdleIntervalMilliseconds = @@ -234,10 +235,11 @@ public long createBootstrap() { enableDNSCache, dnsCacheSaveIntervalSeconds, dnsNumRetries, enableDrainPostDnsRefresh, enableHttp3, useCares, forceV6, useGro, http3ConnectionOptions, http3ClientConnectionOptions, quicHints, quicSuffixes, enableGzipDecompression, - enableBrotliDecompression, enablePortMigration, enableSocketTagging, enableInterfaceBinding, - h2ConnectionKeepaliveIdleIntervalMilliseconds, h2ConnectionKeepaliveTimeoutSeconds, - maxConnectionsPerHost, streamIdleTimeoutSeconds, perTryIdleTimeoutSeconds, appVersion, - appId, enforceTrustChainVerification, filterChain, enablePlatformCertificatesValidation, - upstreamTlsSni, runtimeGuards, caresFallbackResolvers); + enableBrotliDecompression, numTimeoutsToTriggerPortMigration, enableSocketTagging, + enableInterfaceBinding, h2ConnectionKeepaliveIdleIntervalMilliseconds, + h2ConnectionKeepaliveTimeoutSeconds, maxConnectionsPerHost, streamIdleTimeoutSeconds, + perTryIdleTimeoutSeconds, appVersion, appId, enforceTrustChainVerification, filterChain, + enablePlatformCertificatesValidation, upstreamTlsSni, runtimeGuards, + caresFallbackResolvers); } } diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java index fc728abdba0f..2791ae0cd970 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java @@ -307,10 +307,11 @@ public static native long createBootstrap( boolean forceV6, boolean useGro, String http3ConnectionOptions, String http3ClientConnectionOptions, byte[][] quicHints, byte[][] quicCanonicalSuffixes, boolean enableGzipDecompression, boolean enableBrotliDecompression, - boolean enablePortMigration, boolean enableSocketTagging, boolean enableInterfaceBinding, - long h2ConnectionKeepaliveIdleIntervalMilliseconds, long h2ConnectionKeepaliveTimeoutSeconds, - long maxConnectionsPerHost, long streamIdleTimeoutSeconds, long perTryIdleTimeoutSeconds, - String appVersion, String appId, boolean trustChainVerification, byte[][] filterChain, + int numTimeoutsToTriggerPortMigration, boolean enableSocketTagging, + boolean enableInterfaceBinding, long h2ConnectionKeepaliveIdleIntervalMilliseconds, + long h2ConnectionKeepaliveTimeoutSeconds, long maxConnectionsPerHost, + long streamIdleTimeoutSeconds, long perTryIdleTimeoutSeconds, String appVersion, String appId, + boolean trustChainVerification, byte[][] filterChain, boolean enablePlatformCertificatesValidation, String upstreamTlsSni, byte[][] runtimeGuards, byte[][] cares_fallback_resolvers); diff --git a/mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java index bbd867c55b3e..940eb1285447 100644 --- a/mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java @@ -62,7 +62,7 @@ final static class Pkp { private String mQuicClientConnectionOptions = ""; private boolean mHttp2Enabled; private boolean mBrotiEnabled; - private boolean mPortMigrationEnabled; + private int mNumTimeoutsToTriggerPortMigration; private boolean mDisableCache; private int mHttpCacheMode; private long mHttpCacheMaxSize; @@ -240,12 +240,13 @@ public CronvoyEngineBuilderImpl addQuicCanonicalSuffix(String suffix) { List quicCanonicalSuffixes() { return mQuicCanonicalSuffixes; } - public CronvoyEngineBuilderImpl enablePortMigration(boolean enablePortMigration) { - mPortMigrationEnabled = enablePortMigration; + public CronvoyEngineBuilderImpl + setNumTimeoutsToTriggerPortMigration(int numTimeoutsToTriggerPortMigration) { + mNumTimeoutsToTriggerPortMigration = numTimeoutsToTriggerPortMigration; return this; } - boolean portMigrationEnabled() { return mPortMigrationEnabled; } + int numTimeoutsToTriggerPortMigration() { return mNumTimeoutsToTriggerPortMigration; } @Override public CronvoyEngineBuilderImpl addPublicKeyPins(String hostName, Set pinsSha256, diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java index 8cdaed009a57..4ff3415d67a5 100644 --- a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java @@ -291,11 +291,12 @@ private EnvoyConfiguration createEnvoyConfiguration() { mDnsPreresolveHostnames, mEnableDNSCache, mDnsCacheSaveIntervalSeconds, mDnsNumRetries.orElse(-1), mEnableDrainPostDnsRefresh, quicEnabled(), mUseCares, mForceV6, mUseGro, quicConnectionOptions(), quicClientConnectionOptions(), quicHints(), - quicCanonicalSuffixes(), mEnableGzipDecompression, brotliEnabled(), portMigrationEnabled(), - mEnableSocketTag, mEnableInterfaceBinding, mH2ConnectionKeepaliveIdleIntervalMilliseconds, - mH2ConnectionKeepaliveTimeoutSeconds, mMaxConnectionsPerHost, mStreamIdleTimeoutSeconds, - mPerTryIdleTimeoutSeconds, mAppVersion, mAppId, mTrustChainVerification, nativeFilterChain, - platformFilterChain, stringAccessors, keyValueStores, mRuntimeGuards, - mEnablePlatformCertificatesValidation, mUpstreamTlsSni, mCaresFallbackResolvers); + quicCanonicalSuffixes(), mEnableGzipDecompression, brotliEnabled(), + numTimeoutsToTriggerPortMigration(), mEnableSocketTag, mEnableInterfaceBinding, + mH2ConnectionKeepaliveIdleIntervalMilliseconds, mH2ConnectionKeepaliveTimeoutSeconds, + mMaxConnectionsPerHost, mStreamIdleTimeoutSeconds, mPerTryIdleTimeoutSeconds, mAppVersion, + mAppId, mTrustChainVerification, nativeFilterChain, platformFilterChain, stringAccessors, + keyValueStores, mRuntimeGuards, mEnablePlatformCertificatesValidation, mUpstreamTlsSni, + mCaresFallbackResolvers); } } diff --git a/mobile/library/jni/jni_impl.cc b/mobile/library/jni/jni_impl.cc index fd8d0de69ccb..ec987e863e12 100644 --- a/mobile/library/jni/jni_impl.cc +++ b/mobile/library/jni/jni_impl.cc @@ -1222,8 +1222,9 @@ void configureBuilder(Envoy::JNI::JniHelper& jni_helper, jlong connect_timeout_s jboolean use_gro, jstring http3_connection_options, jstring http3_client_connection_options, jobjectArray quic_hints, jobjectArray quic_canonical_suffixes, jboolean enable_gzip_decompression, - jboolean enable_brotli_decompression, jboolean enable_port_migration, - jboolean enable_socket_tagging, jboolean enable_interface_binding, + jboolean enable_brotli_decompression, + jlong num_timeouts_to_trigger_port_migration, jboolean enable_socket_tagging, + jboolean enable_interface_binding, jlong h2_connection_keepalive_idle_interval_milliseconds, jlong h2_connection_keepalive_timeout_seconds, jlong max_connections_per_host, jlong stream_idle_timeout_seconds, jlong per_try_idle_timeout_seconds, @@ -1270,7 +1271,7 @@ void configureBuilder(Envoy::JNI::JniHelper& jni_helper, jlong connect_timeout_s for (const std::string& suffix : suffixes) { builder.addQuicCanonicalSuffix(suffix); } - builder.enablePortMigration(enable_port_migration); + builder.setNumTimeoutsToTriggerPortMigration(num_timeouts_to_trigger_port_migration); builder.setUseCares(use_cares == JNI_TRUE); if (use_cares == JNI_TRUE) { auto resolvers = javaObjectArrayToStringPairVector(jni_helper, cares_fallback_resolvers); @@ -1326,7 +1327,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr jboolean force_v6, jboolean use_gro, jstring http3_connection_options, jstring http3_client_connection_options, jobjectArray quic_hints, jobjectArray quic_canonical_suffixes, jboolean enable_gzip_decompression, - jboolean enable_brotli_decompression, jboolean enable_port_migration, + jboolean enable_brotli_decompression, jlong num_timeouts_to_trigger_port_migration, jboolean enable_socket_tagging, jboolean enable_interface_binding, jlong h2_connection_keepalive_idle_interval_milliseconds, jlong h2_connection_keepalive_timeout_seconds, jlong max_connections_per_host, @@ -1344,8 +1345,8 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr enable_drain_post_dns_refresh, enable_http3, use_cares, force_v6, use_gro, http3_connection_options, http3_client_connection_options, quic_hints, quic_canonical_suffixes, enable_gzip_decompression, enable_brotli_decompression, - enable_port_migration, enable_socket_tagging, enable_interface_binding, - h2_connection_keepalive_idle_interval_milliseconds, + num_timeouts_to_trigger_port_migration, enable_socket_tagging, + enable_interface_binding, h2_connection_keepalive_idle_interval_milliseconds, h2_connection_keepalive_timeout_seconds, max_connections_per_host, stream_idle_timeout_seconds, per_try_idle_timeout_seconds, app_version, app_id, trust_chain_verification, filter_chain, enable_platform_certificates_validation, diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt index bbc6ff7a5144..ebf81c87113d 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt @@ -49,7 +49,7 @@ open class EngineBuilder() { private var quicCanonicalSuffixes = mutableListOf() private var enableGzipDecompression = true private var enableBrotliDecompression = false - private var enablePortMigration = false + private var numTimeoutsToTriggerPortMigration = 0 private var enableSocketTagging = false private var enableInterfaceBinding = false private var h2ConnectionKeepaliveIdleIntervalMilliseconds = 1 @@ -268,13 +268,14 @@ open class EngineBuilder() { } /** - * Specify whether to do quic port migration or not. Defaults to false. + * Configure QUIC port migration. Defaults to disabled. * - * @param enablePortMigration whether or not to allow quic port migration. + * @param numTimeoutsToTriggerPortMigration number of timeouts to trigger port migration. If 0, + * port migration is disabled. * @return This builder. */ - fun enablePortMigration(enablePortMigration: Boolean): EngineBuilder { - this.enablePortMigration = enablePortMigration + fun setNumTimeoutsToTriggerPortMigration(numTimeoutsToTriggerPortMigration: Int): EngineBuilder { + this.numTimeoutsToTriggerPortMigration = numTimeoutsToTriggerPortMigration return this } @@ -578,7 +579,7 @@ open class EngineBuilder() { quicCanonicalSuffixes, enableGzipDecompression, enableBrotliDecompression, - enablePortMigration, + numTimeoutsToTriggerPortMigration, enableSocketTagging, enableInterfaceBinding, h2ConnectionKeepaliveIdleIntervalMilliseconds, diff --git a/mobile/test/cc/unit/envoy_config_test.cc b/mobile/test/cc/unit/envoy_config_test.cc index df94f22ffde4..f0302a298d12 100644 --- a/mobile/test/cc/unit/envoy_config_test.cc +++ b/mobile/test/cc/unit/envoy_config_test.cc @@ -71,7 +71,7 @@ TEST(TestConfig, ConfigIsApplied) { .addQuicHint("www.def.com", 443) .addQuicCanonicalSuffix(".opq.com") .addQuicCanonicalSuffix(".xyz.com") - .enablePortMigration(true) + .setNumTimeoutsToTriggerPortMigration(4) .addConnectTimeoutSeconds(123) .addDnsRefreshSeconds(456) .addDnsMinRefreshSeconds(567) diff --git a/mobile/test/common/http/client_test.cc b/mobile/test/common/http/client_test.cc index bc7d4a61c7e7..0e15da98cc84 100644 --- a/mobile/test/common/http/client_test.cc +++ b/mobile/test/common/http/client_test.cc @@ -498,6 +498,85 @@ TEST_P(ClientTest, MultipleStreams) { ASSERT_EQ(callbacks_called1.on_complete_calls_, 1); } +TEST_P(ClientTest, MultipleUploads) { + envoy_stream_t stream1 = 1; + envoy_stream_t stream2 = 2; + auto request_data1 = std::make_unique("request body1"); + auto request_data2 = std::make_unique("request body2"); + + // Create a stream, and set up request_decoder_ and response_encoder_ + StreamCallbacksCalled callbacks_called1; + auto stream_callbacks1 = createDefaultStreamCallbacks(callbacks_called1); + createStream(std::move(stream_callbacks1)); + + // Send request headers. + EXPECT_CALL(*request_decoder_, decodeHeaders_(_, false)); + http_client_.sendHeaders(stream1, createDefaultRequestHeaders(), false); + http_client_.sendData(stream1, std::move(request_data1), true); + + // Start stream2. + // Setup EnvoyStreamCallbacks to handle the response headers. + NiceMock request_decoder2; + ON_CALL(request_decoder2, streamInfo()).WillByDefault(ReturnRef(stream_info_)); + ResponseEncoder* response_encoder2{}; + StreamCallbacksCalled callbacks_called2; + auto stream_callbacks2 = createDefaultStreamCallbacks(callbacks_called1); + stream_callbacks2.on_headers_ = [&](const ResponseHeaderMap& headers, bool end_stream, + envoy_stream_intel) -> void { + EXPECT_TRUE(end_stream); + EXPECT_EQ(headers.Status()->value().getStringView(), "200"); + callbacks_called2.on_headers_calls_ = true; + }; + stream_callbacks2.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) -> void { + callbacks_called2.on_complete_calls_++; + }; + + std::vector window_callbacks; + ON_CALL(dispatcher_, createSchedulableCallback).WillByDefault([&](std::function cb) { + Event::SchedulableCallbackPtr scheduler = + dispatcher_.Event::ProvisionalDispatcher::createSchedulableCallback(cb); + window_callbacks.push_back(scheduler.get()); + return scheduler; + }); + + // Create a stream. + ON_CALL(dispatcher_, isThreadSafe()).WillByDefault(Return(true)); + + // Grab the response encoder in order to dispatch responses on the stream. + // Return the request decoder to make sure calls are dispatched to the decoder via the dispatcher + // API. + EXPECT_CALL(*api_listener_, newStreamHandle(_, _)) + .WillOnce(Invoke([&](ResponseEncoder& encoder, bool) -> RequestDecoderHandlePtr { + response_encoder2 = &encoder; + return std::make_unique(request_decoder2); + })); + http_client_.startStream(stream2, std::move(stream_callbacks2), explicit_flow_control_); + + // Send request headers. + EXPECT_CALL(request_decoder2, decodeHeaders_(_, false)); + http_client_.sendHeaders(stream2, createDefaultRequestHeaders(), false); + http_client_.sendData(stream2, std::move(request_data2), true); + + for (auto* callback : window_callbacks) { + EXPECT_TRUE(callback->enabled()); + } + + // Finish stream 2. + EXPECT_CALL(dispatcher_, deferredDelete_(_)); + TestResponseHeaderMapImpl response_headers2{{":status", "200"}}; + response_encoder2->encodeHeaders(response_headers2, true); + ASSERT_EQ(callbacks_called2.on_headers_calls_, 1); + // Ensure that the on_headers on the EnvoyStreamCallbacks was called. + ASSERT_EQ(callbacks_called2.on_complete_calls_, 1); + + // Finish stream 1. + EXPECT_CALL(dispatcher_, deferredDelete_(_)); + TestResponseHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, true); + ASSERT_EQ(callbacks_called1.on_headers_calls_, 1); + ASSERT_EQ(callbacks_called1.on_complete_calls_, 1); +} + TEST_P(ClientTest, EnvoyLocalError) { // Override the on_error default with some custom checks. StreamCallbacksCalled callbacks_called; diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc index d428ebf4e888..756535981459 100644 --- a/mobile/test/common/integration/client_integration_test.cc +++ b/mobile/test/common/integration/client_integration_test.cc @@ -694,7 +694,7 @@ TEST_P(ClientIntegrationTest, InvalidDomainReresolveWithNoAddresses) { true); Network::TestResolver::unblockResolve(); terminal_callback_.waitReady(); - EXPECT_EQ(2, getCounterValue("dns_cache.base_dns_cache.dns_query_attempt")); + EXPECT_LE(2, getCounterValue("dns_cache.base_dns_cache.dns_query_attempt")); } TEST_P(ClientIntegrationTest, ReresolveAndDrain) { diff --git a/mobile/test/common/integration/test_server_interface.cc b/mobile/test/common/integration/test_server_interface.cc index 91dc5f46e74f..c704d1c8f951 100644 --- a/mobile/test/common/integration/test_server_interface.cc +++ b/mobile/test/common/integration/test_server_interface.cc @@ -2,39 +2,91 @@ // NOLINT(namespace-envoy) -static std::shared_ptr strong_test_server_; -static std::weak_ptr weak_test_server_; +static std::shared_ptr strong_test_http_server_; +static std::weak_ptr weak_test_http_server_; +static std::shared_ptr strong_test_proxy_server_; +static std::weak_ptr weak_test_proxy_server_; -static std::shared_ptr test_server() { return weak_test_server_.lock(); } +static std::shared_ptr test_http_server() { + if (strong_test_http_server_ == nullptr) { + return nullptr; + } + return weak_test_http_server_.lock(); +} + +static std::shared_ptr test_proxy_server() { + if (strong_test_proxy_server_ == nullptr) { + return nullptr; + } + return weak_test_proxy_server_.lock(); +} -void start_server(Envoy::TestServerType test_server_type) { - strong_test_server_ = std::make_shared(); - weak_test_server_ = strong_test_server_; +void start_http_server(Envoy::TestServerType test_server_type) { + strong_test_http_server_ = std::make_shared(); + weak_test_http_server_ = strong_test_http_server_; - if (auto server = test_server()) { + ASSERT(test_server_type == Envoy::TestServerType::HTTP1_WITHOUT_TLS || + test_server_type == Envoy::TestServerType::HTTP1_WITH_TLS || + test_server_type == Envoy::TestServerType::HTTP2_WITH_TLS || + test_server_type == Envoy::TestServerType::HTTP3, + "Cannot start a proxy server with start_http_server. Use start_proxy_server instead."); + if (auto server = test_http_server()) { server->start(test_server_type, 0); } } -void shutdown_server() { - // Reset the primary handle to the test_server, +void start_proxy_server(Envoy::TestServerType test_server_type) { + strong_test_proxy_server_ = std::make_shared(); + weak_test_proxy_server_ = strong_test_proxy_server_; + + ASSERT(test_server_type == Envoy::TestServerType::HTTP_PROXY || + test_server_type == Envoy::TestServerType::HTTPS_PROXY, + "Cannot start a HTTP server with start_proxy_server. Use start_http_server instead."); + if (auto server = test_proxy_server()) { + server->start(test_server_type, 0); + } +} + +void shutdown_http_server() { + if (strong_test_http_server_ == nullptr) { + return; + } + // Reset the primary handle to the test_http_server, // but retain it long enough to synchronously shutdown. - auto server = strong_test_server_; - strong_test_server_.reset(); + auto server = strong_test_http_server_; + strong_test_http_server_.reset(); server->shutdown(); } -int get_server_port() { - if (auto server = test_server()) { +void shutdown_proxy_server() { + if (strong_test_proxy_server_ == nullptr) { + return; + } + // Reset the primary handle to the test_proxy_server, + // but retain it long enough to synchronously shutdown. + auto server = strong_test_proxy_server_; + strong_test_proxy_server_.reset(); + server->shutdown(); +} + +int get_http_server_port() { + if (auto server = test_http_server()) { + return server->getPort(); + } + return -1; // failure +} + +int get_proxy_server_port() { + if (auto server = test_proxy_server()) { return server->getPort(); } return -1; // failure } -void set_headers_and_data(absl::string_view header_key, absl::string_view header_value, - absl::string_view response_body) { - if (auto server = test_server()) { - // start_server() must be called before headers and data can be added. +void set_http_headers_and_data(absl::string_view header_key, absl::string_view header_value, + absl::string_view response_body) { + if (auto server = test_http_server()) { + // start_http_server() must be called before headers and data can be added. ASSERT(server); server->setHeadersAndData(header_key, header_value, response_body); } diff --git a/mobile/test/common/integration/test_server_interface.h b/mobile/test/common/integration/test_server_interface.h index 0b51d2a6e55f..21e43b722bf3 100644 --- a/mobile/test/common/integration/test_server_interface.h +++ b/mobile/test/common/integration/test_server_interface.h @@ -9,28 +9,47 @@ extern "C" { // functions #endif /** - * Starts the server. Can only have one server active per JVM. This is blocking until the port can - * start accepting requests. + * Starts the HTTP server. Can only have one HTTP server active per process. This is blocking + * until the port can start accepting requests. */ -void start_server(Envoy::TestServerType test_server_type); +void start_http_server(Envoy::TestServerType test_server_type); /** - * Shutdowns the server. Can be restarted later. This is blocking until the server has freed all - * resources. + * Starts the Proxy server. Can only have one proxy server active per process. This is blocking + * until the port can start accepting requests. */ -void shutdown_server(); +void start_proxy_server(Envoy::TestServerType test_server_type); /** - * Returns the port that got attributed. Can only be called once the server has been started. + * Shuts down the HTTP server. Can be restarted later. This is blocking until the server has freed + * all resources. */ -int get_server_port(); +void shutdown_http_server(); /** - * Set response data for server to return for any URL. Can only be called once the server has been - * started. + * Shuts down the proxy server. Can be restarted later. This is blocking until the server has freed + * all resources. */ -void set_headers_and_data(absl::string_view header_key, absl::string_view header_value, - absl::string_view response_body); +void shutdown_proxy_server(); + +/** + * Returns the port that got attributed to the HTTP server. Can only be called once the server has + * been started. + */ +int get_http_server_port(); + +/** + * Returns the port that got attributed to the proxy server. Can only be called once the server has + * been started. + */ +int get_proxy_server_port(); + +/** + * Set response data for the HTTP server to return for any URL. Can only be called once the server + * has been started. + */ +void set_http_headers_and_data(absl::string_view header_key, absl::string_view header_value, + absl::string_view response_body); #ifdef __cplusplus } // functions diff --git a/mobile/test/java/integration/AndroidEngineExplicitFlowTest.java b/mobile/test/java/integration/AndroidEngineExplicitFlowTest.java index e6e19862933a..b29cbce25659 100644 --- a/mobile/test/java/integration/AndroidEngineExplicitFlowTest.java +++ b/mobile/test/java/integration/AndroidEngineExplicitFlowTest.java @@ -481,8 +481,8 @@ public void post_multipleRequests_randomBehavior() throws Exception { mockWebServer.enqueue(new MockResponse().setBody("hello, world")); RequestScenario requestScenario = new RequestScenario() - .setHttpMethod(RequestMethod.GET) - .setUrl(mockWebServer.url("get/flowers").toString()) + .setHttpMethod(RequestMethod.POST) + .setUrl(mockWebServer.url("post/flowers").toString()) .addBody("This is my body part 1") .addBody("This is my body part 2") .setResponseBufferSize(20); // Larger than the response body size diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt index 276072825e8a..e5e621b205a3 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt @@ -92,7 +92,7 @@ class EnvoyConfigurationTest { quicCanonicalSuffixes: MutableList = mutableListOf(".opq.com", ".xyz.com"), enableGzipDecompression: Boolean = true, enableBrotliDecompression: Boolean = false, - enablePortMigration: Boolean = true, + numTimeoutsToTriggerPortMigration: Int = 4, enableSocketTagging: Boolean = false, enableInterfaceBinding: Boolean = false, h2ConnectionKeepaliveIdleIntervalMilliseconds: Int = 222, @@ -140,7 +140,7 @@ class EnvoyConfigurationTest { quicCanonicalSuffixes, enableGzipDecompression, enableBrotliDecompression, - enablePortMigration, + numTimeoutsToTriggerPortMigration, enableSocketTagging, enableInterfaceBinding, h2ConnectionKeepaliveIdleIntervalMilliseconds, diff --git a/mobile/test/java/org/chromium/net/BUILD b/mobile/test/java/org/chromium/net/BUILD index 3b4b49a444f0..43b1efe7d76a 100644 --- a/mobile/test/java/org/chromium/net/BUILD +++ b/mobile/test/java/org/chromium/net/BUILD @@ -305,7 +305,7 @@ envoy_mobile_android_test( srcs = [ "BidirectionalStreamTest.java", ], - flaky = True, # TODO(fredyw): Debug the reason for it being flaky. + flaky = True, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ diff --git a/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java b/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java index 7f56f1c55bb3..53d53a23f92c 100644 --- a/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java +++ b/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java @@ -78,9 +78,6 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { assertTrue(Http2TestServer.shutdownHttp2TestServer()); - if (mCronetEngine != null) { - mCronetEngine.shutdown(); - } } private static void checkResponseInfo(UrlResponseInfo responseInfo, String expectedUrl, diff --git a/mobile/test/kotlin/integration/proxying/BUILD b/mobile/test/kotlin/integration/proxying/BUILD index 6f6866cf287f..b49932636b7d 100644 --- a/mobile/test/kotlin/integration/proxying/BUILD +++ b/mobile/test/kotlin/integration/proxying/BUILD @@ -130,9 +130,6 @@ envoy_mobile_android_test( srcs = [ "ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt", ], - exec_properties = { - "dockerNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_and_listener_extensions.so", ] + select({ @@ -148,5 +145,6 @@ envoy_mobile_android_test( "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", "//test/java/io/envoyproxy/envoymobile/engine/testing", "//test/java/io/envoyproxy/envoymobile/engine/testing:http_proxy_test_server_factory_lib", + "//test/java/io/envoyproxy/envoymobile/engine/testing:http_test_server_factory_lib", ], ) diff --git a/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt b/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt index c80206598d84..6cfc8ce170df 100644 --- a/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt +++ b/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt @@ -12,6 +12,7 @@ import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod import io.envoyproxy.envoymobile.engine.JniLibrary import io.envoyproxy.envoymobile.engine.testing.HttpProxyTestServerFactory +import io.envoyproxy.envoymobile.engine.testing.HttpTestServerFactory import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit @@ -42,16 +43,32 @@ class ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest { } private lateinit var httpProxyTestServer: HttpProxyTestServerFactory.HttpProxyTestServer + private lateinit var httpTestServer: HttpTestServerFactory.HttpTestServer @Before fun setUp() { httpProxyTestServer = HttpProxyTestServerFactory.start(HttpProxyTestServerFactory.Type.HTTP_PROXY) + httpTestServer = + HttpTestServerFactory.start( + HttpTestServerFactory.Type.HTTP1_WITHOUT_TLS, + 0, + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Proxy_servers_and_tunneling/Proxy_Auto-Configuration_PAC_file + mapOf("Content-Type" to "application/x-ns-proxy-autoconfig"), + """ +function FindProxyForURL(url, host) { + return "PROXY ${httpProxyTestServer.ipAddress}:${httpProxyTestServer.port}"; +} + """ + .trimIndent(), + mapOf() + ) } @After fun tearDown() { httpProxyTestServer.shutdown() + httpTestServer.shutdown() } @Test @@ -63,7 +80,7 @@ class ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest { Shadows.shadowOf(connectivityManager) .setProxyForNetwork( connectivityManager.activeNetwork, - ProxyInfo.buildPacProxy(Uri.parse("https://example.com")) + ProxyInfo.buildPacProxy(Uri.parse("http://${httpTestServer.address}")) ) val onEngineRunningLatch = CountDownLatch(1) @@ -85,8 +102,8 @@ class ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest { RequestHeadersBuilder( method = RequestMethod.GET, scheme = "http", - authority = "api.lyft.com", - path = "/ping" + authority = httpTestServer.address, + path = "/" ) .build() @@ -95,7 +112,7 @@ class ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest { .newStreamPrototype() .setOnResponseHeaders { responseHeaders, _, _ -> val status = responseHeaders.httpStatus ?: 0L - assertThat(status).isEqualTo(301) + assertThat(status).isEqualTo(200) assertThat(responseHeaders.value("x-proxy-response")).isNull() onResponseHeadersLatch.countDown() } diff --git a/mobile/test/objective-c/EnvoyTestServer.h b/mobile/test/objective-c/EnvoyTestServer.h index dbc9758d9e64..8d51cf67dde7 100644 --- a/mobile/test/objective-c/EnvoyTestServer.h +++ b/mobile/test/objective-c/EnvoyTestServer.h @@ -10,17 +10,21 @@ // (https://docs.engflow.com/re/client/platform-options-reference.html#sandboxallowed). @interface EnvoyTestServer : NSObject -// Get the port of the upstream server. -+ (NSInteger)getEnvoyPort; +// Get the port of the upstream HTTP server. ++ (NSInteger)getHttpPort; +// Get the port of the upstream proxy server. ++ (NSInteger)getProxyPort; // Starts a server with HTTP1 and no TLS. + (void)startHttp1PlaintextServer; // Starts a server as a HTTP proxy. + (void)startHttpProxyServer; // Starts a server as a HTTPS proxy. + (void)startHttpsProxyServer; -// Shut down and clean up server. -+ (void)shutdownTestServer; -// Add response data to the upstream. +// Shut down and clean up the HTTP server. ++ (void)shutdownTestHttpServer; +// Shut down and clean up the Proxy server. ++ (void)shutdownTestProxyServer; +// Add response data to the HTTP server. + (void)setHeadersAndData:(NSString *)header_key header_value:(NSString *)header_value response_body:(NSString *)response_body; diff --git a/mobile/test/objective-c/EnvoyTestServer.mm b/mobile/test/objective-c/EnvoyTestServer.mm index e1d64714fd52..f07f70796420 100644 --- a/mobile/test/objective-c/EnvoyTestServer.mm +++ b/mobile/test/objective-c/EnvoyTestServer.mm @@ -3,31 +3,39 @@ @implementation EnvoyTestServer -+ (NSInteger)getEnvoyPort { - return get_server_port(); ++ (NSInteger)getHttpPort { + return get_http_server_port(); +} + ++ (NSInteger)getProxyPort { + return get_proxy_server_port(); } + (void)startHttp1PlaintextServer { - start_server(Envoy::TestServerType::HTTP1_WITHOUT_TLS); + start_http_server(Envoy::TestServerType::HTTP1_WITHOUT_TLS); } + (void)startHttpProxyServer { - start_server(Envoy::TestServerType::HTTP_PROXY); + start_proxy_server(Envoy::TestServerType::HTTP_PROXY); } + (void)startHttpsProxyServer { - start_server(Envoy::TestServerType::HTTPS_PROXY); + start_proxy_server(Envoy::TestServerType::HTTPS_PROXY); +} + ++ (void)shutdownTestHttpServer { + shutdown_http_server(); } -+ (void)shutdownTestServer { - shutdown_server(); ++ (void)shutdownTestProxyServer { + shutdown_proxy_server(); } + (void)setHeadersAndData:(NSString *)header_key header_value:(NSString *)header_value response_body:(NSString *)response_body { - set_headers_and_data([header_key UTF8String], [header_value UTF8String], - [response_body UTF8String]); + set_http_headers_and_data([header_key UTF8String], [header_value UTF8String], + [response_body UTF8String]); } @end diff --git a/mobile/test/swift/integration/CancelGRPCStreamTest.swift b/mobile/test/swift/integration/CancelGRPCStreamTest.swift index 6b35c002f08c..2ee52fa2085f 100644 --- a/mobile/test/swift/integration/CancelGRPCStreamTest.swift +++ b/mobile/test/swift/integration/CancelGRPCStreamTest.swift @@ -70,7 +70,7 @@ final class CancelGRPCStreamTests: XCTestCase { let requestHeaders = GRPCRequestHeadersBuilder( scheme: "http", - authority: "localhost:" + String(EnvoyTestServer.getEnvoyPort()), + authority: "localhost:" + String(EnvoyTestServer.getHttpPort()), path: "/") .build() @@ -87,6 +87,6 @@ final class CancelGRPCStreamTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: expectations, timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/CancelStreamTest.swift b/mobile/test/swift/integration/CancelStreamTest.swift index a53f165b2c4b..44314123fca5 100644 --- a/mobile/test/swift/integration/CancelStreamTest.swift +++ b/mobile/test/swift/integration/CancelStreamTest.swift @@ -68,7 +68,7 @@ final class CancelStreamTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/") .build() @@ -86,6 +86,6 @@ final class CancelStreamTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: expectations, timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/EndToEndNetworkingTest.swift b/mobile/test/swift/integration/EndToEndNetworkingTest.swift index 638eefc62333..15e89d65017a 100644 --- a/mobile/test/swift/integration/EndToEndNetworkingTest.swift +++ b/mobile/test/swift/integration/EndToEndNetworkingTest.swift @@ -23,7 +23,7 @@ final class EndToEndNetworkingTest: XCTestCase { "x-response-foo", header_value: "aaa", response_body: "hello world") let headersExpectation = self.expectation(description: "Response headers received") let dataExpectation = self.expectation(description: "Response data received") - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder( method: .get, scheme: "http", authority: "localhost:" + port, path: "/" ) @@ -60,6 +60,6 @@ final class EndToEndNetworkingTest: XCTestCase { XCTAssertEqual(.completed, XCTWaiter().wait(for: expectations, timeout: 10, enforceOrder: true)) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/FilterResetIdleTest.swift b/mobile/test/swift/integration/FilterResetIdleTest.swift index da7f5a528829..7f59c6634056 100644 --- a/mobile/test/swift/integration/FilterResetIdleTest.swift +++ b/mobile/test/swift/integration/FilterResetIdleTest.swift @@ -122,7 +122,7 @@ final class FilterResetIdleTests: XCTestCase { cancelExpectation.isInverted = true EnvoyTestServer.startHttp1PlaintextServer() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let engine = EngineBuilder() .setLogLevel(.debug) @@ -166,6 +166,6 @@ final class FilterResetIdleTests: XCTestCase { ) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/GRPCReceiveErrorTest.swift b/mobile/test/swift/integration/GRPCReceiveErrorTest.swift index a1266db281c7..99efaaac3d74 100644 --- a/mobile/test/swift/integration/GRPCReceiveErrorTest.swift +++ b/mobile/test/swift/integration/GRPCReceiveErrorTest.swift @@ -86,7 +86,7 @@ final class GRPCReceiveErrorTests: XCTestCase { let requestHeaders = GRPCRequestHeadersBuilder( scheme: "http", - authority: "localhost:" + String(EnvoyTestServer.getEnvoyPort()), + authority: "localhost:" + String(EnvoyTestServer.getHttpPort()), path: "/pb.api.v1.Foo/GetBar") .build() let message = Data([1, 2, 3, 4, 5]) @@ -114,6 +114,6 @@ final class GRPCReceiveErrorTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: expectations, timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/IdleTimeoutTest.swift b/mobile/test/swift/integration/IdleTimeoutTest.swift index 95324f9195cd..9735c1202436 100644 --- a/mobile/test/swift/integration/IdleTimeoutTest.swift +++ b/mobile/test/swift/integration/IdleTimeoutTest.swift @@ -95,7 +95,7 @@ final class IdleTimeoutTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder( method: .get, scheme: "http", authority: "localhost:" + port, path: "/" ) @@ -119,6 +119,6 @@ final class IdleTimeoutTests: XCTestCase { ) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/KeyValueStoreTest.swift b/mobile/test/swift/integration/KeyValueStoreTest.swift index 595bae134513..d29363572eea 100644 --- a/mobile/test/swift/integration/KeyValueStoreTest.swift +++ b/mobile/test/swift/integration/KeyValueStoreTest.swift @@ -66,7 +66,7 @@ final class KeyValueStoreTests: XCTestCase { let requestHeaders = RequestHeadersBuilder( method: .get, scheme: "http", - authority: "localhost:" + String(EnvoyTestServer.getEnvoyPort()), path: "/simple.txt" + authority: "localhost:" + String(EnvoyTestServer.getHttpPort()), path: "/simple.txt" ) .build() @@ -84,6 +84,6 @@ final class KeyValueStoreTests: XCTestCase { ) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/ReceiveDataTest.swift b/mobile/test/swift/integration/ReceiveDataTest.swift index 483c947e7261..0ac82ce0c814 100644 --- a/mobile/test/swift/integration/ReceiveDataTest.swift +++ b/mobile/test/swift/integration/ReceiveDataTest.swift @@ -33,7 +33,7 @@ final class ReceiveDataTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -67,6 +67,6 @@ final class ReceiveDataTests: XCTestCase { XCTAssertEqual(actualResponseBody, directResponseBody) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/ResetConnectivityStateTest.swift b/mobile/test/swift/integration/ResetConnectivityStateTest.swift index ddeb5bb0590e..51e357c88205 100644 --- a/mobile/test/swift/integration/ResetConnectivityStateTest.swift +++ b/mobile/test/swift/integration/ResetConnectivityStateTest.swift @@ -30,7 +30,7 @@ final class ResetConnectivityStateTest: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -98,6 +98,6 @@ final class ResetConnectivityStateTest: XCTestCase { XCTAssertTrue(resultEndStream2) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SendDataTest.swift b/mobile/test/swift/integration/SendDataTest.swift index 4a7b736b6f1a..62040a432fdd 100644 --- a/mobile/test/swift/integration/SendDataTest.swift +++ b/mobile/test/swift/integration/SendDataTest.swift @@ -41,7 +41,7 @@ final class SendDataTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -67,6 +67,6 @@ final class SendDataTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [expectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SendHeadersTest.swift b/mobile/test/swift/integration/SendHeadersTest.swift index 46de840b92ef..07c624760756 100644 --- a/mobile/test/swift/integration/SendHeadersTest.swift +++ b/mobile/test/swift/integration/SendHeadersTest.swift @@ -33,7 +33,7 @@ final class SendHeadersTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -62,6 +62,6 @@ final class SendHeadersTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: expectations, timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SendTrailersTest.swift b/mobile/test/swift/integration/SendTrailersTest.swift index 5f1caa7a2404..ca6750935140 100644 --- a/mobile/test/swift/integration/SendTrailersTest.swift +++ b/mobile/test/swift/integration/SendTrailersTest.swift @@ -47,7 +47,7 @@ final class SendTrailersTests: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -73,6 +73,6 @@ final class SendTrailersTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [expectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SetEventTrackerTest.swift b/mobile/test/swift/integration/SetEventTrackerTest.swift index beec655a1717..d3b8c6742682 100644 --- a/mobile/test/swift/integration/SetEventTrackerTest.swift +++ b/mobile/test/swift/integration/SetEventTrackerTest.swift @@ -42,7 +42,7 @@ final class SetEventTrackerTest: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -55,6 +55,6 @@ final class SetEventTrackerTest: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [eventExpectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SetEventTrackerTestNoTracker.swift b/mobile/test/swift/integration/SetEventTrackerTestNoTracker.swift index 33f5035191e0..d37ce770465d 100644 --- a/mobile/test/swift/integration/SetEventTrackerTestNoTracker.swift +++ b/mobile/test/swift/integration/SetEventTrackerTestNoTracker.swift @@ -36,7 +36,7 @@ final class SetEventTrackerTestNoTracker: XCTestCase { let client = engine.streamClient() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "http", authority: "localhost:" + port, path: "/simple.txt") .build() @@ -52,6 +52,6 @@ final class SetEventTrackerTestNoTracker: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [expectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/SetLoggerTest.swift b/mobile/test/swift/integration/SetLoggerTest.swift index 96836ebf9dd0..0d31beac5b36 100644 --- a/mobile/test/swift/integration/SetLoggerTest.swift +++ b/mobile/test/swift/integration/SetLoggerTest.swift @@ -25,7 +25,7 @@ final class LoggerTests: XCTestCase { description: "Run received log event via event tracker") EnvoyTestServer.startHttp1PlaintextServer() - let port = String(EnvoyTestServer.getEnvoyPort()) + let port = String(EnvoyTestServer.getHttpPort()) let engine = EngineBuilder() .setLogLevel(.debug) @@ -64,6 +64,6 @@ final class LoggerTests: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [logEventExpectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestHttpServer() } } diff --git a/mobile/test/swift/integration/proxying/HTTPRequestUsingProxyTest.swift b/mobile/test/swift/integration/proxying/HTTPRequestUsingProxyTest.swift index 67f3e215bb3f..48f5d17a089b 100644 --- a/mobile/test/swift/integration/proxying/HTTPRequestUsingProxyTest.swift +++ b/mobile/test/swift/integration/proxying/HTTPRequestUsingProxyTest.swift @@ -53,7 +53,7 @@ final class HTTPRequestUsingProxyTest: XCTestCase { func testHTTPRequestUsingProxy() throws { EnvoyTestServer.startHttpProxyServer() - let port = EnvoyTestServer.getEnvoyPort() + let port = EnvoyTestServer.getProxyPort() let engineExpectation = self.expectation(description: "Run started engine") @@ -77,13 +77,13 @@ final class HTTPRequestUsingProxyTest: XCTestCase { } engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestProxyServer() } // https://github.com/envoyproxy/envoy/issues/33014 func skipped_testHTTPSRequestUsingProxy() throws { EnvoyTestServer.startHttpsProxyServer() - let port = EnvoyTestServer.getEnvoyPort() + let port = EnvoyTestServer.getProxyPort() let engineExpectation = self.expectation(description: "Run started engine") let responseHeadersExpectation = @@ -136,13 +136,13 @@ final class HTTPRequestUsingProxyTest: XCTestCase { } engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestProxyServer() } // https://github.com/envoyproxy/envoy/issues/33014 func skipped_testHTTPSRequestUsingPacFileUrlResolver() throws { EnvoyTestServer.startHttpsProxyServer() - let port = EnvoyTestServer.getEnvoyPort() + let port = EnvoyTestServer.getProxyPort() let engineExpectation = self.expectation(description: "Run started engine") let responseHeadersExpectation = @@ -195,12 +195,12 @@ final class HTTPRequestUsingProxyTest: XCTestCase { } engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestProxyServer() } func testTwoHTTPRequestsUsingProxy() throws { EnvoyTestServer.startHttpProxyServer() - let port = EnvoyTestServer.getEnvoyPort() + let port = EnvoyTestServer.getProxyPort() let engineExpectation = self.expectation(description: "Run started engine") @@ -227,12 +227,12 @@ final class HTTPRequestUsingProxyTest: XCTestCase { } engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestProxyServer() } func testHTTPRequestUsingProxyCancelStream() throws { EnvoyTestServer.startHttpProxyServer() - let port = EnvoyTestServer.getEnvoyPort() + let port = EnvoyTestServer.getProxyPort() let engineExpectation = self.expectation(description: "Run started engine") @@ -271,7 +271,7 @@ final class HTTPRequestUsingProxyTest: XCTestCase { XCTAssertEqual(XCTWaiter.wait(for: [cancelExpectation], timeout: 10), .completed) engine.terminate() - EnvoyTestServer.shutdownTestServer() + EnvoyTestServer.shutdownTestProxyServer() } // TODO(abeyad): Add test for proxy system settings updated. diff --git a/source/common/common/cleanup.h b/source/common/common/cleanup.h index 3d34bf51b805..bdefb1cbb29f 100644 --- a/source/common/common/cleanup.h +++ b/source/common/common/cleanup.h @@ -24,6 +24,10 @@ class Cleanup { bool cancelled() { return cancelled_; } + static Cleanup Noop() { + return Cleanup([] {}); + } + private: std::function f_; bool cancelled_{false}; diff --git a/source/common/config/null_grpc_mux_impl.h b/source/common/config/null_grpc_mux_impl.h index 453d723eb32d..ce45640bfa20 100644 --- a/source/common/config/null_grpc_mux_impl.h +++ b/source/common/config/null_grpc_mux_impl.h @@ -27,6 +27,12 @@ class NullGrpcMuxImpl : public GrpcMux, ENVOY_BUG(false, "unexpected request for on demand update"); } + absl::Status updateMuxSource(Grpc::RawAsyncClientPtr&&, Grpc::RawAsyncClientPtr&&, + CustomConfigValidatorsPtr&&, Stats::Scope&, BackOffStrategyPtr&&, + const envoy::config::core::v3::ApiConfigSource&) override { + return absl::UnimplementedError(""); + } + EdsResourcesCacheOptRef edsResourcesCache() override { return absl::nullopt; } void onWriteable() override {} diff --git a/source/common/config/utility.cc b/source/common/config/utility.cc index 50a058aa7961..65999f91b6c1 100644 --- a/source/common/config/utility.cc +++ b/source/common/config/utility.cc @@ -346,5 +346,28 @@ Utility::buildJitteredExponentialBackOffStrategy( default_base_interval_ms, default_base_interval_ms * 10, random); } +absl::Status Utility::validateTerminalFilters(const std::string& name, + const std::string& filter_type, + const std::string& filter_chain_type, + bool is_terminal_filter, + bool last_filter_in_current_config) { + if (is_terminal_filter && !last_filter_in_current_config) { + return absl::InvalidArgumentError( + fmt::format("Error: terminal filter named {} of type {} must be the " + "last filter in a {} filter chain.", + name, filter_type, filter_chain_type)); + } else if (!is_terminal_filter && last_filter_in_current_config) { + absl::string_view extra = ""; + if (filter_chain_type == "router upstream http") { + extra = " When upstream_http_filters are specified, they must explicitly end with an " + "UpstreamCodec filter."; + } + return absl::InvalidArgumentError(fmt::format("Error: non-terminal filter named {} of type " + "{} is the last filter in a {} filter chain.{}", + name, filter_type, filter_chain_type, extra)); + } + return absl::OkStatus(); +} + } // namespace Config } // namespace Envoy diff --git a/source/common/config/utility.h b/source/common/config/utility.h index ae9702984bd2..9467b753c7e6 100644 --- a/source/common/config/utility.h +++ b/source/common/config/utility.h @@ -424,19 +424,7 @@ class Utility { const std::string& filter_type, const std::string& filter_chain_type, bool is_terminal_filter, - bool last_filter_in_current_config) { - if (is_terminal_filter && !last_filter_in_current_config) { - return absl::InvalidArgumentError( - fmt::format("Error: terminal filter named {} of type {} must be the " - "last filter in a {} filter chain.", - name, filter_type, filter_chain_type)); - } else if (!is_terminal_filter && last_filter_in_current_config) { - return absl::InvalidArgumentError(fmt::format( - "Error: non-terminal filter named {} of type {} is the last filter in a {} filter chain.", - name, filter_type, filter_chain_type)); - } - return absl::OkStatus(); - } + bool last_filter_in_current_config); /** * Prepares the DNS failure refresh backoff strategy given the cluster configuration. diff --git a/source/common/formatter/http_formatter_context.cc b/source/common/formatter/http_formatter_context.cc index 82d7095f4bba..cf1e9678232b 100644 --- a/source/common/formatter/http_formatter_context.cc +++ b/source/common/formatter/http_formatter_context.cc @@ -49,10 +49,10 @@ static constexpr absl::string_view DEFAULT_FORMAT = "\"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" " "\"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n"; -FormatterPtr HttpSubstitutionFormatUtils::defaultSubstitutionFormatter() { +absl::StatusOr HttpSubstitutionFormatUtils::defaultSubstitutionFormatter() { // It is possible that failed to parse the default format string if the required formatters // are compiled out. - return std::make_unique(DEFAULT_FORMAT, false); + return Envoy::Formatter::FormatterImpl::create(DEFAULT_FORMAT, false); } } // namespace Formatter diff --git a/source/common/formatter/http_formatter_context.h b/source/common/formatter/http_formatter_context.h index 82e08c875412..7009a67f95f9 100644 --- a/source/common/formatter/http_formatter_context.h +++ b/source/common/formatter/http_formatter_context.h @@ -11,7 +11,7 @@ namespace Formatter { */ class HttpSubstitutionFormatUtils { public: - static FormatterPtr defaultSubstitutionFormatter(); + static absl::StatusOr defaultSubstitutionFormatter(); }; } // namespace Formatter diff --git a/source/common/formatter/stream_info_formatter.cc b/source/common/formatter/stream_info_formatter.cc index c7f532db0098..5ff679a731fd 100644 --- a/source/common/formatter/stream_info_formatter.cc +++ b/source/common/formatter/stream_info_formatter.cc @@ -192,7 +192,6 @@ FilterStateFormatter::FilterStateFormatter(absl::string_view key, absl::optional if (!field_name.empty()) { format_ = FilterStateFormat::Field; field_name_ = std::string(field_name); - factory_ = Registry::FactoryRegistry::getFactory(key); } else if (serialize_as_string) { format_ = FilterStateFormat::String; } else { @@ -264,14 +263,7 @@ FilterStateFormatter::format(const StreamInfo::StreamInfo& stream_info) const { #endif } case FilterStateFormat::Field: { - if (!factory_) { - return absl::nullopt; - } - const auto reflection = factory_->reflect(state); - if (!reflection) { - return absl::nullopt; - } - auto field_value = reflection->getField(field_name_); + auto field_value = state->getField(field_name_); auto string_value = absl::visit(StringFieldVisitor(), field_value); if (!string_value) { return absl::nullopt; @@ -315,14 +307,7 @@ FilterStateFormatter::formatValue(const StreamInfo::StreamInfo& stream_info) con return SubstitutionFormatUtils::unspecifiedValue(); } case FilterStateFormat::Field: { - if (!factory_) { - return SubstitutionFormatUtils::unspecifiedValue(); - } - const auto reflection = factory_->reflect(state); - if (!reflection) { - return SubstitutionFormatUtils::unspecifiedValue(); - } - auto field_value = reflection->getField(field_name_); + auto field_value = state->getField(field_name_); auto string_value = absl::visit(StringFieldVisitor(), field_value); if (!string_value) { return SubstitutionFormatUtils::unspecifiedValue(); diff --git a/source/common/formatter/stream_info_formatter.h b/source/common/formatter/stream_info_formatter.h index 0cc80e0d911c..10f4288c9d19 100644 --- a/source/common/formatter/stream_info_formatter.h +++ b/source/common/formatter/stream_info_formatter.h @@ -110,7 +110,6 @@ class FilterStateFormatter : public StreamInfoFormatterProvider { const bool is_upstream_; FilterStateFormat format_; std::string field_name_; - StreamInfo::FilterState::ObjectFactory* factory_; }; class CommonDurationFormatter : public StreamInfoFormatterProvider { diff --git a/source/common/formatter/substitution_format_string.h b/source/common/formatter/substitution_format_string.h index 3a03ee3cfc08..85bbde6750b9 100644 --- a/source/common/formatter/substitution_format_string.h +++ b/source/common/formatter/substitution_format_string.h @@ -65,8 +65,8 @@ class SubstitutionFormatStringUtils { RETURN_IF_NOT_OK_REF(commands.status()); switch (config.format_case()) { case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormat: - return std::make_unique>( - config.text_format(), config.omit_empty_values(), *commands); + return FormatterBaseImpl::create(config.text_format(), + config.omit_empty_values(), *commands); case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kJsonFormat: return createJsonFormatter( config.json_format(), true, config.omit_empty_values(), @@ -76,8 +76,8 @@ class SubstitutionFormatStringUtils { auto data_source_or_error = Config::DataSource::read(config.text_format_source(), true, context.serverFactoryContext().api()); RETURN_IF_NOT_OK(data_source_or_error.status()); - return std::make_unique>( - *data_source_or_error, config.omit_empty_values(), *commands); + return FormatterBaseImpl::create(*data_source_or_error, + config.omit_empty_values(), *commands); } case envoy::config::core::v3::SubstitutionFormatString::FormatCase::FORMAT_NOT_SET: PANIC_DUE_TO_PROTO_UNSET; diff --git a/source/common/formatter/substitution_formatter.h b/source/common/formatter/substitution_formatter.h index f33f07471f48..3915f58a08ea 100644 --- a/source/common/formatter/substitution_formatter.h +++ b/source/common/formatter/substitution_formatter.h @@ -100,7 +100,7 @@ class StreamInfoFormatterWrapper : public FormatterProviderBase - static std::vector> + static absl::StatusOr>> parse(absl::string_view format, const std::vector>& command_parsers = {}) { std::string current_token; @@ -137,7 +137,7 @@ class SubstitutionFormatParser { if (!re2::RE2::Consume(&sub_format, commandWithArgsRegex(), &command, &command_arg, &max_len)) { - throwEnvoyExceptionOrPanic( + return absl::InvalidArgumentError( fmt::format("Incorrect configuration: {}. Couldn't find valid command at position {}", format, pos)); } @@ -184,7 +184,8 @@ class SubstitutionFormatParser { } if (!added) { - throwEnvoyExceptionOrPanic(fmt::format("Not supported field in StreamInfo: {}", command)); + return absl::InvalidArgumentError( + fmt::format("Not supported field in StreamInfo: {}", command)); } pos += (sub_format_size - sub_format.size()); @@ -213,16 +214,14 @@ template class FormatterBaseImpl : public FormatterBase public: using CommandParsers = std::vector>; - FormatterBaseImpl(absl::string_view format, bool omit_empty_values = false) - : empty_value_string_(omit_empty_values ? absl::string_view{} - : DefaultUnspecifiedValueStringView) { - providers_ = SubstitutionFormatParser::parse(format); - } - FormatterBaseImpl(absl::string_view format, bool omit_empty_values, - const CommandParsers& command_parsers) - : empty_value_string_(omit_empty_values ? absl::string_view{} - : DefaultUnspecifiedValueStringView) { - providers_ = SubstitutionFormatParser::parse(format, command_parsers); + static absl::StatusOr> + create(absl::string_view format, bool omit_empty_values = false, + const CommandParsers& command_parsers = {}) { + absl::Status creation_status = absl::OkStatus(); + auto ret = std::unique_ptr( + new FormatterBaseImpl(creation_status, format, omit_empty_values, command_parsers)); + RETURN_IF_NOT_OK_REF(creation_status); + return ret; } // FormatterBase @@ -239,6 +238,25 @@ template class FormatterBaseImpl : public FormatterBase return log_line; } +protected: + FormatterBaseImpl(absl::Status& creation_status, absl::string_view format, + bool omit_empty_values = false) + : empty_value_string_(omit_empty_values ? absl::string_view{} + : DefaultUnspecifiedValueStringView) { + auto providers_or_error = SubstitutionFormatParser::parse(format); + SET_AND_RETURN_IF_NOT_OK(providers_or_error.status(), creation_status); + providers_ = std::move(*providers_or_error); + } + FormatterBaseImpl(absl::Status& creation_status, absl::string_view format, bool omit_empty_values, + const CommandParsers& command_parsers = {}) + : empty_value_string_(omit_empty_values ? absl::string_view{} + : DefaultUnspecifiedValueStringView) { + auto providers_or_error = + SubstitutionFormatParser::parse(format, command_parsers); + SET_AND_RETURN_IF_NOT_OK(providers_or_error.status(), creation_status); + providers_ = std::move(*providers_or_error); + } + private: const std::string empty_value_string_; std::vector> providers_; @@ -387,8 +405,9 @@ class JsonFormatterImplBase : public FormatterBase { for (JsonFormatBuilder::FormatElement& element : JsonFormatBuilder().fromStruct(struct_format)) { if (element.is_template_) { - parsed_elements_.emplace_back( - SubstitutionFormatParser::parse(element.value_, commands)); + parsed_elements_.emplace_back(THROW_OR_RETURN_VALUE( + SubstitutionFormatParser::parse(element.value_, commands), + std::vector>)); } else { parsed_elements_.emplace_back(std::move(element.value_)); } @@ -565,7 +584,7 @@ template class StructFormatterBase { class FormatBuilder { public: explicit FormatBuilder(const CommandParsers& commands) : commands_(commands) {} - std::vector> + absl::StatusOr>> toFormatStringValue(const std::string& string_format) const { return SubstitutionFormatParser::parse(string_format, commands_); } @@ -580,7 +599,9 @@ template class StructFormatterBase { for (const auto& pair : struct_format.fields()) { switch (pair.second.kind_case()) { case ProtobufWkt::Value::kStringValue: - output->emplace(pair.first, toFormatStringValue(pair.second.string_value())); + output->emplace(pair.first, THROW_OR_RETURN_VALUE( + toFormatStringValue(pair.second.string_value()), + std::vector>)); break; case ProtobufWkt::Value::kStructValue: @@ -608,7 +629,9 @@ template class StructFormatterBase { for (const auto& value : list_value_format.values()) { switch (value.kind_case()) { case ProtobufWkt::Value::kStringValue: - output->emplace_back(toFormatStringValue(value.string_value())); + output->emplace_back( + THROW_OR_RETURN_VALUE(toFormatStringValue(value.string_value()), + std::vector>)); break; case ProtobufWkt::Value::kStructValue: diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index e06bca9c6a80..84e358b01c41 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1416,6 +1416,7 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt traceRequest(); } + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); if (!connection_manager_.shouldDeferRequestProxyingToNextIoCycle()) { filter_manager_.decodeHeaders(*request_headers_, end_stream); } else { @@ -1487,6 +1488,7 @@ void ConnectionManagerImpl::ActiveStream::traceRequest() { void ConnectionManagerImpl::ActiveStream::decodeData(Buffer::Instance& data, bool end_stream) { ScopeTrackerScopeState scope(this, connection_manager_.read_callbacks_->connection().dispatcher()); + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); maybeRecordLastByteReceived(end_stream); filter_manager_.streamInfo().addBytesReceived(data.length()); if (!state_.deferred_to_next_io_iteration_) { @@ -1504,6 +1506,7 @@ void ConnectionManagerImpl::ActiveStream::decodeTrailers(RequestTrailerMapPtr&& ENVOY_STREAM_LOG(debug, "request trailers complete:\n{}", *this, *trailers); ScopeTrackerScopeState scope(this, connection_manager_.read_callbacks_->connection().dispatcher()); + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); resetIdleTimer(); ASSERT(!request_trailers_); @@ -1524,6 +1527,7 @@ void ConnectionManagerImpl::ActiveStream::decodeTrailers(RequestTrailerMapPtr&& } void ConnectionManagerImpl::ActiveStream::decodeMetadata(MetadataMapPtr&& metadata_map) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); resetIdleTimer(); if (!state_.deferred_to_next_io_iteration_) { // After going through filters, the ownership of metadata_map will be passed to terminal filter. @@ -1728,6 +1732,7 @@ void ConnectionManagerImpl::ActiveStream::onLocalReply(Code code) { } void ConnectionManagerImpl::ActiveStream::encode1xxHeaders(ResponseHeaderMap& response_headers) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); // Strip the T-E headers etc. Defer other header additions as well as drain-close logic to the // continuation headers. ConnectionManagerUtility::mutateResponseHeaders( @@ -1746,6 +1751,7 @@ void ConnectionManagerImpl::ActiveStream::encode1xxHeaders(ResponseHeaderMap& re void ConnectionManagerImpl::ActiveStream::encodeHeaders(ResponseHeaderMap& headers, bool end_stream) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); // Base headers. // We want to preserve the original date header, but we add a date header if it is absent @@ -1891,6 +1897,7 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ResponseHeaderMap& heade } void ConnectionManagerImpl::ActiveStream::encodeData(Buffer::Instance& data, bool end_stream) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); ENVOY_STREAM_LOG(trace, "encoding data via codec (size={} end_stream={})", *this, data.length(), end_stream); @@ -1899,12 +1906,14 @@ void ConnectionManagerImpl::ActiveStream::encodeData(Buffer::Instance& data, boo } void ConnectionManagerImpl::ActiveStream::encodeTrailers(ResponseTrailerMap& trailers) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); ENVOY_STREAM_LOG(debug, "encoding trailers via codec:\n{}", *this, trailers); response_encoder_->encodeTrailers(trailers); } void ConnectionManagerImpl::ActiveStream::encodeMetadata(MetadataMapPtr&& metadata) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); MetadataMapVector metadata_map_vector; metadata_map_vector.emplace_back(std::move(metadata)); ENVOY_STREAM_LOG(debug, "encoding metadata via codec:\n{}", *this, metadata_map_vector); @@ -2182,6 +2191,7 @@ void ConnectionManagerImpl::ActiveStream::onRequestDataTooLarge() { void ConnectionManagerImpl::ActiveStream::recreateStream( StreamInfo::FilterStateSharedPtr filter_state) { + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); ResponseEncoder* response_encoder = response_encoder_; response_encoder_ = nullptr; @@ -2253,6 +2263,7 @@ bool ConnectionManagerImpl::ActiveStream::onDeferredRequestProcessing() { if (!state_.deferred_to_next_io_iteration_) { return false; } + ENVOY_EXECUTION_SCOPE(trackedStream(), active_span_.get()); state_.deferred_to_next_io_iteration_ = false; bool end_stream = state_.deferred_end_stream_ && deferred_data_ == nullptr && deferred_request_trailers_ == nullptr && deferred_metadata_.empty(); diff --git a/source/common/http/filter_manager.cc b/source/common/http/filter_manager.cc index 35066ddb8806..d158ea808804 100644 --- a/source/common/http/filter_manager.cc +++ b/source/common/http/filter_manager.cc @@ -542,6 +542,7 @@ void FilterManager::decodeHeaders(ActiveStreamDecoderFilter* filter, RequestHead (*entry)->end_stream_); for (; entry != decoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); ASSERT(!(state_.filter_call_state_ & FilterCallState::DecodeHeaders)); state_.filter_call_state_ |= FilterCallState::DecodeHeaders; (*entry)->end_stream_ = (end_stream && continue_data_entry == decoder_filters_.end()); @@ -653,6 +654,7 @@ void FilterManager::decodeData(ActiveStreamDecoderFilter* filter, Buffer::Instan (*entry)->end_stream_); for (; entry != decoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame types, return now. if (handleDataIfStopAll(**entry, data, state_.decoder_filters_streaming_)) { return; @@ -803,6 +805,7 @@ void FilterManager::decodeTrailers(ActiveStreamDecoderFilter* filter, RequestTra ASSERT(!state_.decoder_filter_chain_complete_ || entry == decoder_filters_.end()); for (; entry != decoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame type, return now. if ((*entry)->stoppedAll()) { return; @@ -846,6 +849,7 @@ void FilterManager::decodeMetadata(ActiveStreamDecoderFilter* filter, MetadataMa ASSERT(!(state_.filter_call_state_ & FilterCallState::DecodeMetadata)); for (; entry != decoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame type, stores metadata and returns. // If the filter pointed by entry hasn't returned from decodeHeaders, stores newly added // metadata in case decodeHeaders returns StopAllIteration. The latter can happen when headers @@ -1173,6 +1177,7 @@ void FilterManager::encode1xxHeaders(ActiveStreamEncoderFilter* filter, std::list::iterator entry = commonEncodePrefix(filter, false, FilterIterationStartState::AlwaysStartFromNext); for (; entry != encoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); ASSERT(!(state_.filter_call_state_ & FilterCallState::Encode1xxHeaders)); state_.filter_call_state_ |= FilterCallState::Encode1xxHeaders; const Filter1xxHeadersStatus status = (*entry)->handle_->encode1xxHeaders(headers); @@ -1222,6 +1227,7 @@ void FilterManager::encodeHeaders(ActiveStreamEncoderFilter* filter, ResponseHea std::list::iterator continue_data_entry = encoder_filters_.end(); for (; entry != encoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); ASSERT(!(state_.filter_call_state_ & FilterCallState::EncodeHeaders)); state_.filter_call_state_ |= FilterCallState::EncodeHeaders; (*entry)->end_stream_ = (end_stream && continue_data_entry == encoder_filters_.end()); @@ -1306,6 +1312,7 @@ void FilterManager::encodeMetadata(ActiveStreamEncoderFilter* filter, commonEncodePrefix(filter, false, FilterIterationStartState::CanStartFromCurrent); for (; entry != encoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame type, stores metadata and returns. // If the filter pointed by entry hasn't returned from encodeHeaders, stores newly added // metadata in case encodeHeaders returns StopAllIteration. The latter can happen when headers @@ -1394,6 +1401,7 @@ void FilterManager::encodeData(ActiveStreamEncoderFilter* filter, Buffer::Instan const bool trailers_exists_at_start = filter_manager_callbacks_.responseTrailers().has_value(); for (; entry != encoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame type, return now. if (handleDataIfStopAll(**entry, data, state_.encoder_filters_streaming_)) { return; @@ -1464,6 +1472,7 @@ void FilterManager::encodeTrailers(ActiveStreamEncoderFilter* filter, std::list::iterator entry = commonEncodePrefix(filter, true, FilterIterationStartState::CanStartFromCurrent); for (; entry != encoder_filters_.end(); entry++) { + ENVOY_EXECUTION_SCOPE(trackedStream(), &(*entry)->filter_context_); // If the filter pointed by entry has stopped for all frame type, return now. if ((*entry)->stoppedAll()) { return; @@ -1696,6 +1705,10 @@ bool ActiveStreamDecoderFilter::recreateStream(const ResponseHeaderMap* headers) return false; } + parent_.state_.decoder_filter_chain_aborted_ = true; + parent_.state_.encoder_filter_chain_aborted_ = true; + parent_.state_.recreated_stream_ = true; + parent_.streamInfo().setResponseCodeDetails( StreamInfo::ResponseCodeDetails::get().InternalRedirect); @@ -1762,6 +1775,12 @@ void ActiveStreamEncoderFilter::drainSavedResponseMetadata() { } void ActiveStreamEncoderFilter::handleMetadataAfterHeadersCallback() { + if (parent_.state_.recreated_stream_) { + // The stream has been recreated. In this case, there's no reason to encode saved metadata. + getSavedResponseMetadata()->clear(); + return; + } + // If we drain accumulated metadata, the iteration must start with the current filter. const bool saved_state = iterate_from_current_filter_; iterate_from_current_filter_ = true; diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h index 1798106f864a..94500f13d1c5 100644 --- a/source/common/http/filter_manager.h +++ b/source/common/http/filter_manager.h @@ -894,6 +894,8 @@ class FilterManager : public ScopeTrackedObject, bool decoder_filter_chain_aborted_{}; bool encoder_filter_chain_aborted_{}; bool saw_downstream_reset_{}; + // True when the stream was recreated. + bool recreated_stream_{}; // The following 3 members are booleans rather than part of the space-saving bitfield as they // are passed as arguments to functions expecting bools. Extend State using the bitfield diff --git a/source/common/json/BUILD b/source/common/json/BUILD index 02ae19bfabf0..cd431c83d00c 100644 --- a/source/common/json/BUILD +++ b/source/common/json/BUILD @@ -64,3 +64,13 @@ envoy_cc_library( "@com_google_absl//absl/strings", ], ) + +envoy_cc_library( + name = "json_utility_lib", + srcs = ["json_utility.cc"], + hdrs = ["json_utility.h"], + deps = [ + ":json_streamer_lib", + "//source/common/protobuf:utility_lib_header", + ], +) diff --git a/source/common/json/json_streamer.h b/source/common/json/json_streamer.h index f63d1d439db7..6c3a6790b9a6 100644 --- a/source/common/json/json_streamer.h +++ b/source/common/json/json_streamer.h @@ -456,5 +456,10 @@ template class StreamerBase { */ using BufferStreamer = StreamerBase; +/** + * A Streamer that streams to a string. + */ +using StringStreamer = StreamerBase; + } // namespace Json } // namespace Envoy diff --git a/source/common/json/json_utility.cc b/source/common/json/json_utility.cc new file mode 100644 index 000000000000..d0110e506c31 --- /dev/null +++ b/source/common/json/json_utility.cc @@ -0,0 +1,88 @@ +#include "source/common/json/json_utility.h" + +namespace Envoy { +namespace Json { + +namespace { + +void structValueToJson(const ProtobufWkt::Struct& struct_value, StringStreamer::Map& level); +void listValueToJson(const ProtobufWkt::ListValue& list_value, StringStreamer::Array& level); + +void valueToJson(const ProtobufWkt::Value& value, StringStreamer::Level& level) { + switch (value.kind_case()) { + case ProtobufWkt::Value::KIND_NOT_SET: + case ProtobufWkt::Value::kNullValue: + level.addNull(); + break; + case ProtobufWkt::Value::kNumberValue: + level.addNumber(value.number_value()); + break; + case ProtobufWkt::Value::kStringValue: + level.addString(value.string_value()); + break; + case ProtobufWkt::Value::kBoolValue: + level.addBool(value.bool_value()); + break; + case ProtobufWkt::Value::kStructValue: + structValueToJson(value.struct_value(), *level.addMap()); + break; + case ProtobufWkt::Value::kListValue: + listValueToJson(value.list_value(), *level.addArray()); + break; + } +} + +void structValueToJson(const ProtobufWkt::Struct& struct_value, StringStreamer::Map& map) { + using PairRefWrapper = + std::reference_wrapper::value_type>; + absl::InlinedVector sorted_fields; + sorted_fields.reserve(struct_value.fields_size()); + + for (const auto& field : struct_value.fields()) { + sorted_fields.emplace_back(field); + } + // Sort the keys to make the output deterministic. + std::sort(sorted_fields.begin(), sorted_fields.end(), + [](PairRefWrapper a, PairRefWrapper b) { return a.get().first < b.get().first; }); + + for (const PairRefWrapper field : sorted_fields) { + map.addKey(field.get().first); + valueToJson(field.get().second, map); + } +} + +void listValueToJson(const ProtobufWkt::ListValue& list_value, StringStreamer::Array& arr) { + for (const ProtobufWkt::Value& value : list_value.values()) { + valueToJson(value, arr); + } +} + +} // namespace + +void Utility::appendValueToString(const ProtobufWkt::Value& value, std::string& dest) { + StringStreamer streamer(dest); + switch (value.kind_case()) { + case ProtobufWkt::Value::KIND_NOT_SET: + case ProtobufWkt::Value::kNullValue: + streamer.addNull(); + break; + case ProtobufWkt::Value::kNumberValue: + streamer.addNumber(value.number_value()); + break; + case ProtobufWkt::Value::kStringValue: + streamer.addString(value.string_value()); + break; + case ProtobufWkt::Value::kBoolValue: + streamer.addBool(value.bool_value()); + break; + case ProtobufWkt::Value::kStructValue: + structValueToJson(value.struct_value(), *streamer.makeRootMap()); + break; + case ProtobufWkt::Value::kListValue: + listValueToJson(value.list_value(), *streamer.makeRootArray()); + break; + } +} + +} // namespace Json +} // namespace Envoy diff --git a/source/common/json/json_utility.h b/source/common/json/json_utility.h new file mode 100644 index 000000000000..3797a3befdaa --- /dev/null +++ b/source/common/json/json_utility.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +#include "source/common/json/json_streamer.h" +#include "source/common/protobuf/protobuf.h" + +namespace Envoy { +namespace Json { + +class Utility { +public: + /** + * Convert a ProtobufWkt::Value to a JSON string. + * @param value message of type type.googleapis.com/google.protobuf.Value + * @param dest JSON string. + */ + static void appendValueToString(const ProtobufWkt::Value& value, std::string& dest); +}; + +} // namespace Json +} // namespace Envoy diff --git a/source/common/listener_manager/listener_impl.cc b/source/common/listener_manager/listener_impl.cc index 1e2c8cd03e45..ed1585afe452 100644 --- a/source/common/listener_manager/listener_impl.cc +++ b/source/common/listener_manager/listener_impl.cc @@ -72,12 +72,27 @@ bool shouldBindToPort(const envoy::config::listener::v3::Listener& config) { } } // namespace +absl::StatusOr> ListenSocketFactoryImpl::create( + ListenerComponentFactory& factory, Network::Address::InstanceConstSharedPtr address, + Network::Socket::Type socket_type, const Network::Socket::OptionsSharedPtr& options, + const std::string& listener_name, uint32_t tcp_backlog_size, + ListenerComponentFactory::BindType bind_type, + const Network::SocketCreationOptions& creation_options, uint32_t num_sockets) { + absl::Status creation_status = absl::OkStatus(); + auto ret = std::unique_ptr(new ListenSocketFactoryImpl( + factory, address, socket_type, options, listener_name, tcp_backlog_size, bind_type, + creation_options, num_sockets, creation_status)); + RETURN_IF_NOT_OK(creation_status); + return ret; +} + ListenSocketFactoryImpl::ListenSocketFactoryImpl( ListenerComponentFactory& factory, Network::Address::InstanceConstSharedPtr address, Network::Socket::Type socket_type, const Network::Socket::OptionsSharedPtr& options, const std::string& listener_name, uint32_t tcp_backlog_size, ListenerComponentFactory::BindType bind_type, - const Network::SocketCreationOptions& creation_options, uint32_t num_sockets) + const Network::SocketCreationOptions& creation_options, uint32_t num_sockets, + absl::Status& creation_status) : factory_(factory), local_address_(address), socket_type_(socket_type), options_(options), listener_name_(listener_name), tcp_backlog_size_(tcp_backlog_size), bind_type_(bind_type), socket_creation_options_(creation_options) { @@ -100,8 +115,9 @@ ListenSocketFactoryImpl::ListenSocketFactoryImpl( } } - sockets_.push_back(THROW_OR_RETURN_VALUE( - createListenSocketAndApplyOptions(factory, socket_type, 0), Network::SocketSharedPtr)); + auto socket_or_error = createListenSocketAndApplyOptions(factory, socket_type, 0); + SET_AND_RETURN_IF_NOT_OK(socket_or_error.status(), creation_status); + sockets_.push_back(*socket_or_error); if (sockets_[0] != nullptr && local_address_->ip() && local_address_->ip()->port() == 0) { local_address_ = sockets_[0]->connectionInfoProvider().localAddress(); @@ -114,8 +130,9 @@ ListenSocketFactoryImpl::ListenSocketFactoryImpl( if (bind_type_ != ListenerComponentFactory::BindType::ReusePort && sockets_[0] != nullptr) { sockets_.push_back(sockets_[0]->duplicate()); } else { - sockets_.push_back(THROW_OR_RETURN_VALUE( - createListenSocketAndApplyOptions(factory, socket_type, i), Network::SocketSharedPtr)); + auto socket_or_error = createListenSocketAndApplyOptions(factory, socket_type, i); + SET_AND_RETURN_IF_NOT_OK(socket_or_error.status(), creation_status); + sockets_.push_back(*socket_or_error); } } ASSERT(sockets_.size() == num_sockets); @@ -165,12 +182,7 @@ absl::StatusOr ListenSocketFactoryImpl::createListenSo fmt::format("{}: Setting socket options {}", listener_name_, ok ? "succeeded" : "failed"); if (!ok) { ENVOY_LOG(warn, "{}", message); -#ifdef ENVOY_DISABLE_EXCEPTIONS - PANIC(message); -#else - throw Network::SocketOptionException(message); -#endif - + return absl::InvalidArgumentError(message); } else { ENVOY_LOG(debug, "{}", message); } @@ -236,8 +248,11 @@ std::string listenerStatsScope(const envoy::config::listener::v3::Listener& conf return absl::StrCat("envoy_internal_", config.name()); } auto address_or_error = Network::Address::resolveProtoAddress(config.address()); - THROW_IF_NOT_OK_REF(address_or_error.status()); - return address_or_error.value()->asString(); + if (address_or_error.status().ok()) { + return address_or_error.value()->asString(); + } + // Listener creation will fail shortly when the address is used. + return absl::StrCat("invalid_address_listener"); } } // namespace @@ -254,10 +269,22 @@ Network::DrainDecision& ListenerFactoryContextBaseImpl::drainDecision() { return Server::DrainManager& ListenerFactoryContextBaseImpl::drainManager() { return *drain_manager_; } Init::Manager& ListenerFactoryContextBaseImpl::initManager() { return server_.initManager(); } +absl::StatusOr> +ListenerImpl::create(const envoy::config::listener::v3::Listener& config, + const std::string& version_info, ListenerManagerImpl& parent, + const std::string& name, bool added_via_api, bool workers_started, + uint64_t hash) { + absl::Status creation_status = absl::OkStatus(); + auto ret = std::unique_ptr(new ListenerImpl( + config, version_info, parent, name, added_via_api, workers_started, hash, creation_status)); + RETURN_IF_NOT_OK(creation_status); + return ret; +} + ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, const std::string& version_info, ListenerManagerImpl& parent, const std::string& name, bool added_via_api, bool workers_started, - uint64_t hash) + uint64_t hash, absl::Status& creation_status) : parent_(parent), socket_type_(config.has_internal_listener() ? Network::Socket::Type::Stream @@ -324,25 +351,28 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, // All the addresses should be same socket type, so get the first address's socket type is // enough. auto address_or_error = Network::Address::resolveProtoAddress(config.address()); - THROW_IF_NOT_OK_REF(address_or_error.status()); + SET_AND_RETURN_IF_NOT_OK(address_or_error.status(), creation_status); auto address = std::move(address_or_error.value()); - THROW_IF_NOT_OK(checkIpv4CompatAddress(address, config.address())); + SET_AND_RETURN_IF_NOT_OK(checkIpv4CompatAddress(address, config.address()), creation_status); addresses_.emplace_back(address); address_opts_list.emplace_back(std::ref(config.socket_options())); for (auto i = 0; i < config.additional_addresses_size(); i++) { if (socket_type_ != Network::Utility::protobufAddressSocketType(config.additional_addresses(i).address())) { - throwEnvoyExceptionOrPanic( + creation_status = absl::InvalidArgumentError( fmt::format("listener {}: has different socket type. The listener only " "support same socket type for all the addresses.", name_)); + return; } auto addresses_or_error = Network::Address::resolveProtoAddress(config.additional_addresses(i).address()); - THROW_IF_NOT_OK_REF(addresses_or_error.status()); + SET_AND_RETURN_IF_NOT_OK(addresses_or_error.status(), creation_status); auto additional_address = std::move(addresses_or_error.value()); - THROW_IF_NOT_OK(checkIpv4CompatAddress(address, config.additional_addresses(i).address())); + SET_AND_RETURN_IF_NOT_OK( + checkIpv4CompatAddress(address, config.additional_addresses(i).address()), + creation_status); addresses_.emplace_back(additional_address); if (config.additional_addresses(i).has_socket_options()) { address_opts_list.emplace_back( @@ -367,20 +397,21 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, addresses_, listener_factory_context_->parentFactoryContext(), initManager()), buildAccessLog(config); - THROW_IF_NOT_OK(validateConfig()); + SET_AND_RETURN_IF_NOT_OK(validateConfig(), creation_status); // buildUdpListenerFactory() must come before buildListenSocketOptions() because the UDP // listener factory can provide additional options. - THROW_IF_NOT_OK(buildUdpListenerFactory(config, parent_.server_.options().concurrency())); + SET_AND_RETURN_IF_NOT_OK(buildUdpListenerFactory(config, parent_.server_.options().concurrency()), + creation_status); buildListenSocketOptions(config, address_opts_list); - THROW_IF_NOT_OK(createListenerFilterFactories(config)); - THROW_IF_NOT_OK(validateFilterChains(config)); - THROW_IF_NOT_OK(buildFilterChains(config)); + SET_AND_RETURN_IF_NOT_OK(createListenerFilterFactories(config), creation_status); + SET_AND_RETURN_IF_NOT_OK(validateFilterChains(config), creation_status); + SET_AND_RETURN_IF_NOT_OK(buildFilterChains(config), creation_status); if (socket_type_ != Network::Socket::Type::Datagram) { buildSocketOptions(config); buildOriginalDstListenerFilter(config); buildProxyProtocolListenerFilter(config); - THROW_IF_NOT_OK(buildInternalListener(config)); + SET_AND_RETURN_IF_NOT_OK(buildInternalListener(config), creation_status); } if (!workers_started_) { // Initialize dynamic_init_manager_ from Server's init manager if it's not initialized. @@ -395,7 +426,7 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, const envoy::config::listener::v3::Listener& config, const std::string& version_info, ListenerManagerImpl& parent, const std::string& name, bool added_via_api, bool workers_started, - uint64_t hash) + uint64_t hash, absl::Status& creation_status) : parent_(parent), addresses_(origin.addresses_), socket_type_(origin.socket_type_), bind_to_port_(shouldBindToPort(config)), mptcp_enabled_(config.enable_mptcp()), hand_off_restored_destination_connections_( @@ -443,11 +474,11 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, missing_listener_config_stats_({ALL_MISSING_LISTENER_CONFIG_STATS( POOL_COUNTER(listener_factory_context_->listenerScope()))}) { buildAccessLog(config); - THROW_IF_NOT_OK(validateConfig()); - THROW_IF_NOT_OK(createListenerFilterFactories(config)); - THROW_IF_NOT_OK(validateFilterChains(config)); - THROW_IF_NOT_OK(buildFilterChains(config)); - THROW_IF_NOT_OK(buildInternalListener(config)); + SET_AND_RETURN_IF_NOT_OK(validateConfig(), creation_status); + SET_AND_RETURN_IF_NOT_OK(createListenerFilterFactories(config), creation_status); + SET_AND_RETURN_IF_NOT_OK(validateFilterChains(config), creation_status); + SET_AND_RETURN_IF_NOT_OK(buildFilterChains(config), creation_status); + SET_AND_RETURN_IF_NOT_OK(buildInternalListener(config), creation_status); if (socket_type_ == Network::Socket::Type::Stream) { // Apply the options below only for TCP. buildSocketOptions(config); @@ -677,6 +708,13 @@ void ListenerImpl::buildListenSocketOptions( addListenSocketOptions(listen_socket_options_list_[i], Network::SocketOptionFactory::buildUdpGroOptions()); } + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment")) { + addListenSocketOptions( + listen_socket_options_list_[i], + Network::SocketOptionFactory::buildDoNotFragmentOptions( + /*mapped_v6*/ addresses_[i]->ip()->version() == Network::Address::IpVersion::v6 && + !addresses_[i]->ip()->ipv6()->v6only())); + } // Additional factory specific options. ASSERT(udp_listener_config_->listener_factory_ != nullptr, @@ -1013,14 +1051,18 @@ bool ListenerImpl::supportUpdateFilterChain(const envoy::config::listener::v3::L return false; } -ListenerImplPtr +absl::StatusOr ListenerImpl::newListenerWithFilterChain(const envoy::config::listener::v3::Listener& config, bool workers_started, uint64_t hash) { + + absl::Status creation_status = absl::OkStatus(); // Use WrapUnique since the constructor is private. - return absl::WrapUnique(new ListenerImpl(*this, config, version_info_, parent_, name_, - added_via_api_, - /* new new workers started state */ workers_started, - /* use new hash */ hash)); + auto ret = absl::WrapUnique(new ListenerImpl(*this, config, version_info_, parent_, name_, + added_via_api_, + /* new new workers started state */ workers_started, + /* use new hash */ hash, creation_status)); + RETURN_IF_NOT_OK(creation_status); + return ret; } void ListenerImpl::diffFilterChain(const ListenerImpl& another_listener, diff --git a/source/common/listener_manager/listener_impl.h b/source/common/listener_manager/listener_impl.h index f7bd70f47a3c..2157b5eac533 100644 --- a/source/common/listener_manager/listener_impl.h +++ b/source/common/listener_manager/listener_impl.h @@ -63,14 +63,12 @@ class ListenerManagerImpl; class ListenSocketFactoryImpl : public Network::ListenSocketFactory, protected Logger::Loggable { public: - ListenSocketFactoryImpl(ListenerComponentFactory& factory, - Network::Address::InstanceConstSharedPtr address, - Network::Socket::Type socket_type, - const Network::Socket::OptionsSharedPtr& options, - const std::string& listener_name, uint32_t tcp_backlog_size, - ListenerComponentFactory::BindType bind_type, - const Network::SocketCreationOptions& creation_options, - uint32_t num_sockets); + static absl::StatusOr> + create(ListenerComponentFactory& factory, Network::Address::InstanceConstSharedPtr address, + Network::Socket::Type socket_type, const Network::Socket::OptionsSharedPtr& options, + const std::string& listener_name, uint32_t tcp_backlog_size, + ListenerComponentFactory::BindType bind_type, + const Network::SocketCreationOptions& creation_options, uint32_t num_sockets); // Network::ListenSocketFactory Network::Socket::Type socketType() const override { return socket_type_; } @@ -89,6 +87,15 @@ class ListenSocketFactoryImpl : public Network::ListenSocketFactory, absl::Status doFinalPreWorkerInit() override; private: + ListenSocketFactoryImpl(ListenerComponentFactory& factory, + Network::Address::InstanceConstSharedPtr address, + Network::Socket::Type socket_type, + const Network::Socket::OptionsSharedPtr& options, + const std::string& listener_name, uint32_t tcp_backlog_size, + ListenerComponentFactory::BindType bind_type, + const Network::SocketCreationOptions& creation_options, + uint32_t num_sockets, absl::Status& creation_status); + ListenSocketFactoryImpl(const ListenSocketFactoryImpl& factory_to_clone); absl::StatusOr @@ -205,9 +212,10 @@ class ListenerImpl final : public Network::ListenerConfig, * @param hash supplies the hash to use for duplicate checking. * @param concurrency is the number of listeners instances to be created. */ - ListenerImpl(const envoy::config::listener::v3::Listener& config, const std::string& version_info, - ListenerManagerImpl& parent, const std::string& name, bool added_via_api, - bool workers_started, uint64_t hash); + static absl::StatusOr> + create(const envoy::config::listener::v3::Listener& config, const std::string& version_info, + ListenerManagerImpl& parent, const std::string& name, bool added_via_api, + bool workers_started, uint64_t hash); ~ListenerImpl() override; // TODO(lambdai): Explore using the same ListenerImpl object to execute in place filter chain @@ -216,7 +224,7 @@ class ListenerImpl final : public Network::ListenerConfig, * Execute in place filter chain update. The filter chain update is less expensive than full * listener update because connections may not need to be drained. */ - std::unique_ptr + absl::StatusOr> newListenerWithFilterChain(const envoy::config::listener::v3::Listener& config, bool workers_started, uint64_t hash); /** @@ -344,6 +352,9 @@ class ListenerImpl final : public Network::ListenerConfig, SystemTime last_updated_; private: + ListenerImpl(const envoy::config::listener::v3::Listener& config, const std::string& version_info, + ListenerManagerImpl& parent, const std::string& name, bool added_via_api, + bool workers_started, uint64_t hash, absl::Status& creation_status); struct UdpListenerConfigImpl : public Network::UdpListenerConfig { UdpListenerConfigImpl(const envoy::config::listener::v3::UdpListenerConfig config) : config_(config) {} @@ -384,7 +395,8 @@ class ListenerImpl final : public Network::ListenerConfig, */ ListenerImpl(ListenerImpl& origin, const envoy::config::listener::v3::Listener& config, const std::string& version_info, ListenerManagerImpl& parent, - const std::string& name, bool added_via_api, bool workers_started, uint64_t hash); + const std::string& name, bool added_via_api, bool workers_started, uint64_t hash, + absl::Status& creation_status); // Helpers for constructor. void buildAccessLog(const envoy::config::listener::v3::Listener& config); absl::Status buildInternalListener(const envoy::config::listener::v3::Listener& config); diff --git a/source/common/listener_manager/listener_manager_impl.cc b/source/common/listener_manager/listener_manager_impl.cc index 1683caf51b80..8de7a7da3360 100644 --- a/source/common/listener_manager/listener_manager_impl.cc +++ b/source/common/listener_manager/listener_manager_impl.cc @@ -580,13 +580,17 @@ absl::StatusOr ListenerManagerImpl::addOrUpdateListenerInternal( (*existing_active_listener)->supportUpdateFilterChain(config, workers_started_)) { ENVOY_LOG(debug, "use in place update filter chain update path for listener name={} hash={}", name, hash); - new_listener = + auto listener_or_error = (*existing_active_listener)->newListenerWithFilterChain(config, workers_started_, hash); + RETURN_IF_NOT_OK_REF(listener_or_error.status()); + new_listener = std::move(*listener_or_error); stats_.listener_in_place_updated_.inc(); } else { ENVOY_LOG(debug, "use full listener update path for listener name={} hash={}", name, hash); - new_listener = std::make_unique(config, version_info, *this, name, added_via_api, + auto listener_or_error = ListenerImpl::create(config, version_info, *this, name, added_via_api, workers_started_, hash); + RETURN_IF_NOT_OK_REF(listener_or_error.status()); + new_listener = std::move(*listener_or_error); } ListenerImpl& new_listener_ref = *new_listener; @@ -1197,10 +1201,15 @@ absl::Status ListenerManagerImpl::createListenSocketFactory(ListenerImpl& listen creation_options.mptcp_enabled_ = listener.mptcpEnabled(); for (std::vector::size_type i = 0; i < listener.addresses().size(); i++) { - socket_status = listener.addSocketFactory(std::make_unique( + auto factory_or_error = ListenSocketFactoryImpl::create( *factory_, listener.addresses()[i], socket_type, listener.listenSocketOptions(i), listener.name(), listener.tcpBacklogSize(), bind_type, creation_options, - server_.options().concurrency())); + server_.options().concurrency()); + if (!factory_or_error.status().ok()) { + socket_status = factory_or_error.status(); + } else { + socket_status = listener.addSocketFactory(std::move(*factory_or_error)); + } if (!socket_status.ok()) { break; } diff --git a/source/common/network/filter_state_dst_address.cc b/source/common/network/filter_state_dst_address.cc index b164573e5f7a..231f3e0b1c60 100644 --- a/source/common/network/filter_state_dst_address.cc +++ b/source/common/network/filter_state_dst_address.cc @@ -9,39 +9,25 @@ absl::optional AddressObject::hash() const { return HashUtil::xxHash64(address_->asStringView()); } -class AddressObjectReflection : public StreamInfo::FilterState::ObjectReflection { -public: - AddressObjectReflection(const AddressObject* object) : object_(object) {} - FieldType getField(absl::string_view field_name) const override { - const auto* ip = object_->address_->ip(); - if (!ip) { - return {}; - } - if (field_name == "ip") { - return ip->addressAsString(); - } else if (field_name == "port") { - return int64_t(ip->port()); - } +StreamInfo::FilterState::Object::FieldType +AddressObject::getField(absl::string_view field_name) const { + const auto* ip = address_->ip(); + if (!ip) { return {}; } - -private: - const AddressObject* object_; -}; + if (field_name == "ip") { + return ip->addressAsString(); + } else if (field_name == "port") { + return int64_t(ip->port()); + } + return {}; +} std::unique_ptr BaseAddressObjectFactory::createFromBytes(absl::string_view data) const { const auto address = Utility::parseInternetAddressAndPortNoThrow(std::string(data)); return address ? std::make_unique(address) : nullptr; } -std::unique_ptr -BaseAddressObjectFactory::reflect(const StreamInfo::FilterState::Object* data) const { - const auto* object = dynamic_cast(data); - if (object) { - return std::make_unique(object); - } - return nullptr; -} } // namespace Network } // namespace Envoy diff --git a/source/common/network/filter_state_dst_address.h b/source/common/network/filter_state_dst_address.h index ec6c565fd689..b35cb25b9fc0 100644 --- a/source/common/network/filter_state_dst_address.h +++ b/source/common/network/filter_state_dst_address.h @@ -17,13 +17,15 @@ class AddressObject : public StreamInfo::FilterState::Object, public Hashable { absl::optional serializeAsString() const override { return address_ ? absl::make_optional(address_->asString()) : absl::nullopt; } + bool hasFieldSupport() const override { return true; } + FieldType getField(absl::string_view field_name) const override; + // Implements hashing interface because the value is applied once per upstream connection. // Multiple streams sharing the upstream connection must have the same address object. absl::optional hash() const override; private: const Network::Address::InstanceConstSharedPtr address_; - friend class AddressObjectReflection; }; /** @@ -33,8 +35,6 @@ class BaseAddressObjectFactory : public StreamInfo::FilterState::ObjectFactory { public: std::unique_ptr createFromBytes(absl::string_view data) const override; - std::unique_ptr - reflect(const StreamInfo::FilterState::Object* data) const override; }; } // namespace Network diff --git a/source/common/network/socket_option_factory.cc b/source/common/network/socket_option_factory.cc index 094ebca07423..83fc9bfc3043 100644 --- a/source/common/network/socket_option_factory.cc +++ b/source/common/network/socket_option_factory.cc @@ -181,5 +181,32 @@ std::unique_ptr SocketOptionFactory::buildIpRecvTosOptions() { return options; } +std::unique_ptr +SocketOptionFactory::buildDoNotFragmentOptions(bool supports_v4_mapped_v6_addresses) { + auto options = std::make_unique(); +#ifdef ENVOY_IP_DONTFRAG + options->push_back(std::make_shared( + envoy::config::core::v3::SocketOption::STATE_PREBIND, ENVOY_IP_DONTFRAG, ENVOY_IPV6_DONTFRAG, + 1)); + // v4 mapped v6 addresses don't support ENVOY_IP_DONTFRAG on MAC OS. + (void)supports_v4_mapped_v6_addresses; +#elif defined(ENVOY_IP_MTU_DISCOVER) + options->push_back(std::make_shared( + envoy::config::core::v3::SocketOption::STATE_PREBIND, ENVOY_IP_MTU_DISCOVER, + ENVOY_IP_MTU_DISCOVER_VALUE, ENVOY_IPV6_MTU_DISCOVER, ENVOY_IPV6_MTU_DISCOVER_VALUE)); + + if (supports_v4_mapped_v6_addresses) { + ENVOY_LOG_MISC(trace, "Also apply the V4 option to v6 socket to support v4-mapped addresses."); + options->push_back( + std::make_shared(envoy::config::core::v3::SocketOption::STATE_PREBIND, + ENVOY_IP_MTU_DISCOVER, ENVOY_IP_MTU_DISCOVER_VALUE)); + } +#else + (void)supports_v4_mapped_v6_addresses; + ENVOY_LOG_MISC(trace, "Platform supports neither socket option IP_DONTFRAG nor IP_MTU_DISCOVER"); +#endif + return options; +} + } // namespace Network } // namespace Envoy diff --git a/source/common/network/socket_option_factory.h b/source/common/network/socket_option_factory.h index 90f521bc672b..94bfd5f83912 100644 --- a/source/common/network/socket_option_factory.h +++ b/source/common/network/socket_option_factory.h @@ -38,6 +38,12 @@ class SocketOptionFactory : Logger::Loggable { static std::unique_ptr buildUdpGroOptions(); static std::unique_ptr buildZeroSoLingerOptions(); static std::unique_ptr buildIpRecvTosOptions(); + /** + * @param supports_v4_mapped_v6_addresses true if this option is to be applied to a v6 socket with + * v4-mapped v6 address(i.e. ::ffff:172.21.0.6) support. + */ + static std::unique_ptr + buildDoNotFragmentOptions(bool supports_v4_mapped_v6_addresses); }; } // namespace Network } // namespace Envoy diff --git a/source/common/orca/BUILD b/source/common/orca/BUILD index 79aa43340804..56d750828fb6 100644 --- a/source/common/orca/BUILD +++ b/source/common/orca/BUILD @@ -13,8 +13,11 @@ envoy_cc_library( srcs = ["orca_parser.cc"], hdrs = ["orca_parser.h"], deps = [ + "//envoy/common:exception_lib", "//envoy/http:header_map_interface", "//source/common/common:base64_lib", + "//source/common/http:header_utility_lib", + "//source/common/protobuf:utility_lib_header", "@com_github_cncf_xds//xds/data/orca/v3:pkg_cc_proto", "@com_github_fmtlib_fmt//:fmtlib", "@com_google_absl//absl/status:statusor", diff --git a/source/common/orca/orca_parser.cc b/source/common/orca/orca_parser.cc index a18061bb256c..2dd29bc7944f 100644 --- a/source/common/orca/orca_parser.cc +++ b/source/common/orca/orca_parser.cc @@ -1,13 +1,27 @@ #include "source/common/orca/orca_parser.h" +#include +#include #include +#include +#include +#include "envoy/common/exception.h" #include "envoy/http/header_map.h" #include "source/common/common/base64.h" #include "source/common/common/fmt.h" +#include "source/common/http/header_utility.h" +#include "source/common/protobuf/utility.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#include "absl/strings/strip.h" using ::Envoy::Http::HeaderMap; using xds::data::orca::v3::OrcaLoadReport; @@ -17,29 +31,147 @@ namespace Orca { namespace { +const Http::LowerCaseString& endpointLoadMetricsHeader() { + CONSTRUCT_ON_FIRST_USE(Http::LowerCaseString, kEndpointLoadMetricsHeader); +} + const Http::LowerCaseString& endpointLoadMetricsHeaderBin() { CONSTRUCT_ON_FIRST_USE(Http::LowerCaseString, kEndpointLoadMetricsHeaderBin); } +absl::Status tryCopyNamedMetricToOrcaLoadReport(absl::string_view metric_name, double metric_value, + OrcaLoadReport& orca_load_report) { + if (metric_name.empty()) { + return absl::InvalidArgumentError("named metric key is empty."); + } + + orca_load_report.mutable_named_metrics()->insert({std::string(metric_name), metric_value}); + return absl::OkStatus(); +} + +std::vector parseCommaDelimitedHeader(const absl::string_view entry) { + std::vector values; + std::vector tokens = + Envoy::Http::HeaderUtility::parseCommaDelimitedHeader(entry); + values.insert(values.end(), tokens.begin(), tokens.end()); + return values; +} + +absl::Status tryCopyMetricToOrcaLoadReport(absl::string_view metric_name, + absl::string_view metric_value, + OrcaLoadReport& orca_load_report) { + if (metric_name.empty()) { + return absl::InvalidArgumentError("metric names cannot be empty strings"); + } + + if (metric_value.empty()) { + return absl::InvalidArgumentError("metric values cannot be empty strings"); + } + + double value; + if (!absl::SimpleAtod(metric_value, &value)) { + return absl::InvalidArgumentError(fmt::format( + "unable to parse custom backend load metric value({}): {}", metric_name, metric_value)); + } + + if (std::isnan(value)) { + return absl::InvalidArgumentError( + fmt::format("custom backend load metric value({}) cannot be NaN.", metric_name)); + } + + if (std::isinf(value)) { + return absl::InvalidArgumentError( + fmt::format("custom backend load metric value({}) cannot be infinity.", metric_name)); + } + + if (absl::StartsWith(metric_name, kNamedMetricsFieldPrefix)) { + auto metric_name_without_prefix = absl::StripPrefix(metric_name, kNamedMetricsFieldPrefix); + return tryCopyNamedMetricToOrcaLoadReport(metric_name_without_prefix, value, orca_load_report); + } + + if (metric_name == kCpuUtilizationField) { + orca_load_report.set_cpu_utilization(value); + } else if (metric_name == kMemUtilizationField) { + orca_load_report.set_mem_utilization(value); + } else if (metric_name == kApplicationUtilizationField) { + orca_load_report.set_application_utilization(value); + } else if (metric_name == kEpsField) { + orca_load_report.set_eps(value); + } else if (metric_name == kRpsFractionalField) { + orca_load_report.set_rps_fractional(value); + } else { + return absl::InvalidArgumentError(absl::StrCat("unsupported metric name: ", metric_name)); + } + return absl::OkStatus(); +} + +absl::Status tryParseNativeHttpEncoded(const absl::string_view header, + OrcaLoadReport& orca_load_report) { + const std::vector values = parseCommaDelimitedHeader(header); + + // Check for duplicate metric names here because OrcaLoadReport fields are not + // marked as optional and therefore don't differentiate between unset and + // default values. + absl::flat_hash_set metric_names; + for (const auto value : values) { + std::pair entry = + absl::StrSplit(value, absl::MaxSplits(':', 1), absl::SkipWhitespace()); + if (metric_names.contains(entry.first)) { + return absl::AlreadyExistsError( + absl::StrCat(kEndpointLoadMetricsHeader, " contains duplicate metric: ", entry.first)); + } + RETURN_IF_NOT_OK(tryCopyMetricToOrcaLoadReport(entry.first, entry.second, orca_load_report)); + metric_names.insert(entry.first); + } + return absl::OkStatus(); +} + +absl::Status tryParseSerializedBinary(const absl::string_view header, + OrcaLoadReport& orca_load_report) { + if (header.empty()) { + return absl::InvalidArgumentError("ORCA binary header value is empty"); + } + const std::string decoded_value = Envoy::Base64::decode(header); + if (decoded_value.empty()) { + return absl::InvalidArgumentError( + fmt::format("unable to decode ORCA binary header value: {}", header)); + } + if (!orca_load_report.ParseFromString(decoded_value)) { + return absl::InvalidArgumentError( + fmt::format("unable to parse binaryheader to OrcaLoadReport: {}", header)); + } + return absl::OkStatus(); +} + } // namespace absl::StatusOr parseOrcaLoadReportHeaders(const HeaderMap& headers) { OrcaLoadReport load_report; - // Binary protobuf format. + // Binary protobuf format. Legacy header from gRPC implementation. if (const auto header_bin = headers.get(endpointLoadMetricsHeaderBin()); !header_bin.empty()) { const auto header_value = header_bin[0]->value().getStringView(); - if (header_value.empty()) { - return absl::InvalidArgumentError("ORCA binary header value is empty"); - } - const std::string decoded_value = Envoy::Base64::decode(header_value); - if (decoded_value.empty()) { - return absl::InvalidArgumentError( - fmt::format("unable to decode ORCA binary header value: {}", header_value)); - } - if (!load_report.ParseFromString(decoded_value)) { + RETURN_IF_NOT_OK(tryParseSerializedBinary(header_value, load_report)); + } else if (const auto header = headers.get(endpointLoadMetricsHeader()); !header.empty()) { + std::pair split_header = + absl::StrSplit(header[0]->value().getStringView(), absl::MaxSplits(' ', 1)); + + if (split_header.first == kHeaderFormatPrefixBin) { // Binary protobuf format. + RETURN_IF_NOT_OK(tryParseSerializedBinary(split_header.second, load_report)); + } else if (split_header.first == kHeaderFormatPrefixText) { // Native HTTP format. + RETURN_IF_NOT_OK(tryParseNativeHttpEncoded(split_header.second, load_report)); + } else if (split_header.first == kHeaderFormatPrefixJson) { // JSON format. +#if defined(ENVOY_ENABLE_FULL_PROTOS) && defined(ENVOY_ENABLE_YAML) + const std::string json_string = std::string(split_header.second); + bool has_unknown_field = false; + RETURN_IF_ERROR( + Envoy::MessageUtil::loadFromJsonNoThrow(json_string, load_report, has_unknown_field)); +#else + IS_ENVOY_BUG("JSON formatted ORCA header support not implemented for this build"); +#endif // !ENVOY_ENABLE_FULL_PROTOS || !ENVOY_ENABLE_YAML + } else { return absl::InvalidArgumentError( - fmt::format("unable to parse binaryheader to OrcaLoadReport: {}", header_value)); + fmt::format("unsupported ORCA header format: {}", split_header.first)); } } else { return absl::NotFoundError("no ORCA data sent from the backend"); diff --git a/source/common/orca/orca_parser.h b/source/common/orca/orca_parser.h index 86fd23944017..6c6f4552757c 100644 --- a/source/common/orca/orca_parser.h +++ b/source/common/orca/orca_parser.h @@ -8,11 +8,24 @@ namespace Envoy { namespace Orca { -// Header used to send ORCA load metrics from the backend. +// Headers used to send ORCA load metrics from the backend. +static constexpr absl::string_view kEndpointLoadMetricsHeader = "endpoint-load-metrics"; static constexpr absl::string_view kEndpointLoadMetricsHeaderBin = "endpoint-load-metrics-bin"; +// Prefix used to determine format expected in kEndpointLoadMetricsHeader. +static constexpr absl::string_view kHeaderFormatPrefixBin = "BIN"; +static constexpr absl::string_view kHeaderFormatPrefixJson = "JSON"; +static constexpr absl::string_view kHeaderFormatPrefixText = "TEXT"; +// The following fields are the names of the metrics tracked in the ORCA load +// report proto. +static constexpr absl::string_view kApplicationUtilizationField = "application_utilization"; +static constexpr absl::string_view kCpuUtilizationField = "cpu_utilization"; +static constexpr absl::string_view kMemUtilizationField = "mem_utilization"; +static constexpr absl::string_view kEpsField = "eps"; +static constexpr absl::string_view kRpsFractionalField = "rps_fractional"; +static constexpr absl::string_view kNamedMetricsFieldPrefix = "named_metrics."; // Parses ORCA load metrics from a header map into an OrcaLoadReport proto. -// Supports serialized binary formats. +// Supports native HTTP, JSON and serialized binary formats. absl::StatusOr parseOrcaLoadReportHeaders(const Envoy::Http::HeaderMap& headers); } // namespace Orca diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index 2d427e7d9e7a..b14e3720b6b0 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -255,7 +255,7 @@ class MessageUtil { static std::size_t hash(const Protobuf::Message& message); #ifdef ENVOY_ENABLE_YAML - static void loadFromJson(const std::string& json, Protobuf::Message& message, + static void loadFromJson(absl::string_view json, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor); /** * Return ok only when strict conversion(don't ignore unknown field) succeeds. @@ -264,9 +264,9 @@ class MessageUtil { * Return error status for relaxed conversion and set has_unknown_field to false if relaxed * conversion(ignore unknown field) fails. */ - static absl::Status loadFromJsonNoThrow(const std::string& json, Protobuf::Message& message, + static absl::Status loadFromJsonNoThrow(absl::string_view json, Protobuf::Message& message, bool& has_unknown_fileld); - static void loadFromJson(const std::string& json, ProtobufWkt::Struct& message); + static void loadFromJson(absl::string_view json, ProtobufWkt::Struct& message); static void loadFromYaml(const std::string& yaml, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor); #endif diff --git a/source/common/protobuf/yaml_utility.cc b/source/common/protobuf/yaml_utility.cc index 4409abc33d64..741817bc0431 100644 --- a/source/common/protobuf/yaml_utility.cc +++ b/source/common/protobuf/yaml_utility.cc @@ -115,7 +115,7 @@ void jsonConvertInternal(const Protobuf::Message& source, } // namespace -void MessageUtil::loadFromJson(const std::string& json, Protobuf::Message& message, +void MessageUtil::loadFromJson(absl::string_view json, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor) { bool has_unknown_field; auto load_status = loadFromJsonNoThrow(json, message, has_unknown_field); @@ -124,15 +124,16 @@ void MessageUtil::loadFromJson(const std::string& json, Protobuf::Message& messa } if (has_unknown_field) { // If the parsing failure is caused by the unknown fields. - THROW_IF_NOT_OK(validation_visitor.onUnknownField("type " + message.GetTypeName() + " reason " + - load_status.ToString())); + THROW_IF_NOT_OK(validation_visitor.onUnknownField( + fmt::format("type {} reason {}", message.GetTypeName(), load_status.ToString()))); } else { // If the error has nothing to do with unknown field. - throw EnvoyException("Unable to parse JSON as proto (" + load_status.ToString() + "): " + json); + throw EnvoyException( + fmt::format("Unable to parse JSON as proto ({}): {}", load_status.ToString(), json)); } } -absl::Status MessageUtil::loadFromJsonNoThrow(const std::string& json, Protobuf::Message& message, +absl::Status MessageUtil::loadFromJsonNoThrow(absl::string_view json, Protobuf::Message& message, bool& has_unknown_fileld) { has_unknown_fileld = false; Protobuf::util::JsonParseOptions options; @@ -163,7 +164,7 @@ absl::Status MessageUtil::loadFromJsonNoThrow(const std::string& json, Protobuf: return relaxed_status; } -void MessageUtil::loadFromJson(const std::string& json, ProtobufWkt::Struct& message) { +void MessageUtil::loadFromJson(absl::string_view json, ProtobufWkt::Struct& message) { // No need to validate if converting to a Struct, since there are no unknown // fields possible. loadFromJson(json, message, ProtobufMessage::getNullValidationVisitor()); diff --git a/source/common/quic/envoy_quic_client_session.h b/source/common/quic/envoy_quic_client_session.h index 65e69027bc58..8d3d1a8579a2 100644 --- a/source/common/quic/envoy_quic_client_session.h +++ b/source/common/quic/envoy_quic_client_session.h @@ -75,8 +75,7 @@ class EnvoyQuicClientSession : public QuicFilterManagerConnectionImpl, // PacketsToReadDelegate size_t numPacketsExpectedPerEventLoop() const override { - // Do one round of reading per active stream, or to see if there's a new - // active stream. + // Do one round of reading per active stream, or to see if there's a new active stream. return std::max(1, GetNumActiveStreams()) * Network::NUM_DATAGRAMS_PER_RECEIVE; } diff --git a/source/common/quic/envoy_quic_client_stream.cc b/source/common/quic/envoy_quic_client_stream.cc index 88b1a96d767a..cdfb83d2553f 100644 --- a/source/common/quic/envoy_quic_client_stream.cc +++ b/source/common/quic/envoy_quic_client_stream.cc @@ -258,7 +258,8 @@ bool EnvoyQuicClientStream::OnStopSending(quic::QuicResetStreamError error) { runResetCallbacks( quicRstErrorToEnvoyRemoteResetReason(error.internal_code()), Runtime::runtimeFeatureEnabled("envoy.reloadable_features.report_stream_reset_error_code") - ? quic::QuicRstStreamErrorCodeToString(error.internal_code()) + ? absl::StrCat(quic::QuicRstStreamErrorCodeToString(error.internal_code()), + "|FROM_PEER") : absl::string_view()); } return true; @@ -360,7 +361,7 @@ void EnvoyQuicClientStream::OnStreamReset(const quic::QuicRstStreamFrame& frame) runResetCallbacks( quicRstErrorToEnvoyRemoteResetReason(frame.error_code), Runtime::runtimeFeatureEnabled("envoy.reloadable_features.report_stream_reset_error_code") - ? quic::QuicRstStreamErrorCodeToString(frame.error_code) + ? absl::StrCat(quic::QuicRstStreamErrorCodeToString(frame.error_code), "|FROM_PEER") : absl::string_view()); } } @@ -374,7 +375,7 @@ void EnvoyQuicClientStream::ResetWithError(quic::QuicResetStreamError error) { runResetCallbacks( quicRstErrorToEnvoyLocalResetReason(error.internal_code()), Runtime::runtimeFeatureEnabled("envoy.reloadable_features.report_stream_reset_error_code") - ? quic::QuicRstStreamErrorCodeToString(error.internal_code()) + ? absl::StrCat(quic::QuicRstStreamErrorCodeToString(error.internal_code()), "|FROM_SELF") : absl::string_view()); if (session()->connection()->connected()) { quic::QuicSpdyClientStream::ResetWithError(error); diff --git a/source/common/quic/envoy_quic_server_stream.cc b/source/common/quic/envoy_quic_server_stream.cc index 331495df292b..5da95f50d26c 100644 --- a/source/common/quic/envoy_quic_server_stream.cc +++ b/source/common/quic/envoy_quic_server_stream.cc @@ -340,7 +340,8 @@ bool EnvoyQuicServerStream::OnStopSending(quic::QuicResetStreamError error) { runResetCallbacks( quicRstErrorToEnvoyRemoteResetReason(error.internal_code()), Runtime::runtimeFeatureEnabled("envoy.reloadable_features.report_stream_reset_error_code") - ? quic::QuicRstStreamErrorCodeToString(error.internal_code()) + ? absl::StrCat(quic::QuicRstStreamErrorCodeToString(error.internal_code()), + "|FROM_PEER") : absl::string_view()); } return true; @@ -360,7 +361,7 @@ void EnvoyQuicServerStream::OnStreamReset(const quic::QuicRstStreamFrame& frame) runResetCallbacks( quicRstErrorToEnvoyRemoteResetReason(frame.error_code), Runtime::runtimeFeatureEnabled("envoy.reloadable_features.report_stream_reset_error_code") - ? quic::QuicRstStreamErrorCodeToString(frame.error_code) + ? absl::StrCat(quic::QuicRstStreamErrorCodeToString(frame.error_code), "|FROM_PEER") : absl::string_view()); } } @@ -375,7 +376,8 @@ void EnvoyQuicServerStream::ResetWithError(quic::QuicResetStreamError error) { runResetCallbacks( quicRstErrorToEnvoyLocalResetReason(error.internal_code()), Runtime::runtimeFeatureEnabled("envoy.reloadable_features.report_stream_reset_error_code") - ? quic::QuicRstStreamErrorCodeToString(error.internal_code()) + ? absl::StrCat(quic::QuicRstStreamErrorCodeToString(error.internal_code()), + "|FROM_SELF") : absl::string_view()); } quic::QuicSpdyServerStreamBase::ResetWithError(error); diff --git a/source/common/quic/envoy_quic_utils.cc b/source/common/quic/envoy_quic_utils.cc index 098abd6b795d..28d59302e5c1 100644 --- a/source/common/quic/envoy_quic_utils.cc +++ b/source/common/quic/envoy_quic_utils.cc @@ -17,11 +17,13 @@ namespace Quic { namespace { Network::Address::InstanceConstSharedPtr -getLoopbackAddress(const Network::Address::IpVersion version) { - if (version == Network::Address::IpVersion::v6) { - return std::make_shared("::1"); +getLoopbackAddress(Network::Address::InstanceConstSharedPtr peer_address) { + if (peer_address->ip()->version() == Network::Address::IpVersion::v6) { + return std::make_shared( + "::1", 0, &peer_address->socketInterface(), peer_address->ip()->ipv6()->v6only()); } - return std::make_shared("127.0.0.1"); + return std::make_shared("127.0.0.1", + &peer_address->socketInterface()); } } // namespace @@ -200,7 +202,7 @@ createConnectionSocket(const Network::Address::InstanceConstSharedPtr& peer_addr Network::Socket::Type::Datagram, // Use the loopback address if `local_addr` is null, to pass in the socket interface used to // create the IoHandle, without having to make the more expensive `getifaddrs` call. - local_addr ? local_addr : getLoopbackAddress(peer_addr->ip()->version()), peer_addr, + local_addr ? local_addr : getLoopbackAddress(peer_addr), peer_addr, Network::SocketCreationOptions{false, max_addresses_cache_size}); connection_socket->setDetectedTransportProtocol("quic"); if (!connection_socket->isOpen()) { @@ -215,6 +217,25 @@ createConnectionSocket(const Network::Address::InstanceConstSharedPtr& peer_addr if (prefer_gro && Api::OsSysCallsSingleton::get().supportsUdpGro()) { connection_socket->addOptions(Network::SocketOptionFactory::buildUdpGroOptions()); } + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment")) { + int v6_only = 0; + if (connection_socket->ipVersion().has_value() && + connection_socket->ipVersion().value() == Network::Address::IpVersion::v6) { + socklen_t v6_only_len = sizeof(v6_only); + Api::SysCallIntResult result = + connection_socket->getSocketOption(IPPROTO_IPV6, IPV6_V6ONLY, &v6_only, &v6_only_len); + if (result.return_value_ != 0) { + ENVOY_LOG_MISC( + error, "Failed to get IPV6_V6ONLY socket option, getsockopt() returned {}, errno {}", + result.return_value_, result.errno_); + connection_socket->close(); + return connection_socket; + } + } + connection_socket->addOptions(Network::SocketOptionFactory::buildDoNotFragmentOptions( + /*mapped_v6*/ connection_socket->ipVersion().value() == Network::Address::IpVersion::v6 && + v6_only == 0)); + } if (options != nullptr) { connection_socket->addOptions(options); } diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 0b3ffdaefdc0..78b7c56537ce 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -478,7 +478,6 @@ class RetryPolicyImpl : public RetryPolicy { // that should be used when with this policy. std::vector> retry_host_predicate_configs_; - Upstream::RetryPrioritySharedPtr retry_priority_; // Name and config proto to use to create the RetryPriority to use with this policy. Default // initialized when no RetryPriority should be used. std::pair retry_priority_config_; diff --git a/source/common/router/header_parser.cc b/source/common/router/header_parser.cc index 4d33c0ce4580..f7feb18a632e 100644 --- a/source/common/router/header_parser.cc +++ b/source/common/router/header_parser.cc @@ -45,7 +45,7 @@ parseHttpHeaderFormatter(const envoy::config::core::v3::HeaderValue& header_valu final_header_value = HeaderParser::translatePerRequestState(final_header_value); // Let the substitution formatter parse the final_header_value. - return std::make_unique(final_header_value, true); + return Envoy::Formatter::FormatterImpl::create(final_header_value, true); } } // namespace diff --git a/source/common/router/router_ratelimit.cc b/source/common/router/router_ratelimit.cc index db2dcc45e298..77c285a85a24 100644 --- a/source/common/router/router_ratelimit.cc +++ b/source/common/router/router_ratelimit.cc @@ -12,8 +12,6 @@ #include "source/common/common/empty_string.h" #include "source/common/config/metadata.h" #include "source/common/config/utility.h" -#include "source/common/http/matching/data_impl.h" -#include "source/common/matcher/matcher.h" #include "source/common/protobuf/utility.h" namespace Envoy { @@ -39,44 +37,24 @@ bool populateDescriptor(const std::vector& act return result; } -class RateLimitDescriptorValidationVisitor - : public Matcher::MatchTreeValidationVisitor { -public: - absl::Status performDataInputValidation(const Matcher::DataInputFactory&, - absl::string_view) override { - return absl::OkStatus(); +} // namespace + +// Ratelimit::DescriptorProducer +bool MatchInputRateLimitDescriptor::populateDescriptor(RateLimit::DescriptorEntry& descriptor_entry, + const std::string&, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const { + Http::Matching::HttpMatchingDataImpl data(info); + data.onRequestHeaders(headers); + auto result = data_input_->get(data); + if (!absl::holds_alternative(result.data_)) { + return false; } -}; - -class MatchInputRateLimitDescriptor : public RateLimit::DescriptorProducer { -public: - MatchInputRateLimitDescriptor(const std::string& descriptor_key, - Matcher::DataInputPtr&& data_input) - : descriptor_key_(descriptor_key), data_input_(std::move(data_input)) {} - - // Ratelimit::DescriptorProducer - bool populateDescriptor(RateLimit::DescriptorEntry& descriptor_entry, const std::string&, - const Http::RequestHeaderMap& headers, - const StreamInfo::StreamInfo& info) const override { - Http::Matching::HttpMatchingDataImpl data(info); - data.onRequestHeaders(headers); - auto result = data_input_->get(data); - if (absl::holds_alternative(result.data_)) { - return false; - } - const std::string& str = absl::get(result.data_); - if (!str.empty()) { - descriptor_entry = {descriptor_key_, str}; - } - return true; + if (absl::string_view str = absl::get(result.data_); !str.empty()) { + descriptor_entry = {descriptor_key_, std::string(str)}; } - -private: - const std::string descriptor_key_; - Matcher::DataInputPtr data_input_; -}; - -} // namespace + return true; +} const uint64_t RateLimitPolicyImpl::MAX_STAGE_NUMBER = 10UL; @@ -256,7 +234,8 @@ QueryParameterValueMatchAction::QueryParameterValueMatchAction( : descriptor_value_(action.descriptor_value()), descriptor_key_(!action.descriptor_key().empty() ? action.descriptor_key() : "query_match"), expect_match_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(action, expect_match, true)), - action_query_parameters_(buildQueryParameterMatcherVector(action, context)) {} + action_query_parameters_( + buildQueryParameterMatcherVector(action.query_parameters(), context)) {} bool QueryParameterValueMatchAction::populateDescriptor( RateLimit::DescriptorEntry& descriptor_entry, const std::string&, @@ -274,10 +253,11 @@ bool QueryParameterValueMatchAction::populateDescriptor( std::vector QueryParameterValueMatchAction::buildQueryParameterMatcherVector( - const envoy::config::route::v3::RateLimit::Action::QueryParameterValueMatch& action, + const Protobuf::RepeatedPtrField& + query_parameters, Server::Configuration::CommonFactoryContext& context) { std::vector ret; - for (const auto& query_parameter : action.query_parameters()) { + for (const auto& query_parameter : query_parameters) { ret.push_back(std::make_unique(query_parameter, context)); } return ret; diff --git a/source/common/router/router_ratelimit.h b/source/common/router/router_ratelimit.h index b069af8d7ed6..3fb5149a4cc2 100644 --- a/source/common/router/router_ratelimit.h +++ b/source/common/router/router_ratelimit.h @@ -13,6 +13,8 @@ #include "source/common/config/metadata.h" #include "source/common/http/header_utility.h" +#include "source/common/http/matching/data_impl.h" +#include "source/common/matcher/matcher.h" #include "source/common/network/cidr_range.h" #include "source/common/protobuf/utility.h" #include "source/common/router/config_utility.h" @@ -146,6 +148,7 @@ class MetaDataAction : public RateLimit::DescriptorProducer { MetaDataAction(const envoy::config::route::v3::RateLimit::Action::MetaData& action); // for maintaining backward compatibility with the deprecated DynamicMetaData action MetaDataAction(const envoy::config::route::v3::RateLimit::Action::DynamicMetaData& action); + // Ratelimit::DescriptorProducer bool populateDescriptor(RateLimit::DescriptorEntry& descriptor_entry, const std::string& local_service_cluster, @@ -198,7 +201,8 @@ class QueryParameterValueMatchAction : public RateLimit::DescriptorProducer { const StreamInfo::StreamInfo& info) const override; std::vector buildQueryParameterMatcherVector( - const envoy::config::route::v3::RateLimit::Action::QueryParameterValueMatch& action, + const Protobuf::RepeatedPtrField& + query_parameters, Server::Configuration::CommonFactoryContext& context); private: @@ -208,6 +212,31 @@ class QueryParameterValueMatchAction : public RateLimit::DescriptorProducer { const std::vector action_query_parameters_; }; +class RateLimitDescriptorValidationVisitor + : public Matcher::MatchTreeValidationVisitor { +public: + absl::Status performDataInputValidation(const Matcher::DataInputFactory&, + absl::string_view) override { + return absl::OkStatus(); + } +}; + +class MatchInputRateLimitDescriptor : public RateLimit::DescriptorProducer { +public: + MatchInputRateLimitDescriptor(const std::string& descriptor_key, + Matcher::DataInputPtr&& data_input) + : descriptor_key_(descriptor_key), data_input_(std::move(data_input)) {} + + // Ratelimit::DescriptorProducer + bool populateDescriptor(RateLimit::DescriptorEntry& descriptor_entry, const std::string&, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const override; + +private: + const std::string descriptor_key_; + Matcher::DataInputPtr data_input_; +}; + /* * Implementation of RateLimitPolicyEntry that holds the action for the configuration. */ diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 6ec39cd77dd3..a06e49b182ea 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -94,6 +94,7 @@ RUNTIME_GUARD(envoy_reloadable_features_skip_dns_lookup_for_proxied_requests); RUNTIME_GUARD(envoy_reloadable_features_strict_duration_validation); RUNTIME_GUARD(envoy_reloadable_features_tcp_tunneling_send_downstream_fin_on_upstream_trailers); RUNTIME_GUARD(envoy_reloadable_features_test_feature_true); +RUNTIME_GUARD(envoy_reloadable_features_udp_set_do_not_fragment); RUNTIME_GUARD(envoy_reloadable_features_udp_socket_apply_aggregated_read_limit); RUNTIME_GUARD(envoy_reloadable_features_uhv_allow_malformed_url_encoding); RUNTIME_GUARD(envoy_reloadable_features_upstream_remote_address_use_connection); @@ -105,6 +106,7 @@ RUNTIME_GUARD(envoy_reloadable_features_use_typed_metadata_in_proxy_protocol_lis RUNTIME_GUARD(envoy_reloadable_features_validate_connect); RUNTIME_GUARD(envoy_reloadable_features_validate_grpc_header_before_log_grpc_status); RUNTIME_GUARD(envoy_reloadable_features_validate_upstream_headers); +RUNTIME_GUARD(envoy_reloadable_features_xds_failover_to_primary_enabled); RUNTIME_GUARD(envoy_reloadable_features_xdstp_path_avoid_colon_encoding); RUNTIME_GUARD(envoy_restart_features_allow_client_socket_creation_failure); RUNTIME_GUARD(envoy_restart_features_allow_slot_destroy_on_worker_threads); diff --git a/source/common/stream_info/bool_accessor_impl.h b/source/common/stream_info/bool_accessor_impl.h index 868bfefab953..8de1563fae48 100644 --- a/source/common/stream_info/bool_accessor_impl.h +++ b/source/common/stream_info/bool_accessor_impl.h @@ -19,6 +19,10 @@ class BoolAccessorImpl : public BoolAccessor { return message; } + absl::optional serializeAsString() const override { + return value_ ? "true" : "false"; + } + // From BoolAccessor. bool value() const override { return value_; } diff --git a/source/common/stream_info/utility.cc b/source/common/stream_info/utility.cc index d8ad2e5f1d80..35b370602a66 100644 --- a/source/common/stream_info/utility.cc +++ b/source/common/stream_info/utility.cc @@ -200,6 +200,14 @@ absl::optional TimingUtility::lastDownstreamTxByteSent return duration(timing.value().get().lastDownstreamTxByteSent(), stream_info_); } +absl::optional TimingUtility::lastDownstreamHeaderRxByteReceived() { + OptRef timing = stream_info_.downstreamTiming(); + if (!timing) { + return absl::nullopt; + } + return duration(timing.value().get().lastDownstreamHeaderRxByteReceived(), stream_info_); +} + absl::optional TimingUtility::lastDownstreamRxByteReceived() { OptRef timing = stream_info_.downstreamTiming(); if (!timing) { diff --git a/source/common/stream_info/utility.h b/source/common/stream_info/utility.h index 8c8a3d7408f3..d95e6993ff6f 100644 --- a/source/common/stream_info/utility.h +++ b/source/common/stream_info/utility.h @@ -219,6 +219,7 @@ class TimingUtility { absl::optional upstreamHandshakeComplete(); absl::optional firstDownstreamTxByteSent(); absl::optional lastDownstreamTxByteSent(); + absl::optional lastDownstreamHeaderRxByteReceived(); absl::optional lastDownstreamRxByteReceived(); absl::optional downstreamHandshakeComplete(); absl::optional lastDownstreamAckReceived(); diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index c12503b2169c..36fa65196d7e 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -550,10 +550,12 @@ ProdClusterInfoFactory::createClusterInfo(const CreateClusterInfoParams& params) factory_context, socket_factory, *scope), std::unique_ptr); - return std::make_unique( - params.server_context_.initManager(), params.server_context_, params.cluster_, - params.bind_config_, params.server_context_.runtime(), std::move(socket_matcher), - std::move(scope), params.added_via_api_, factory_context); + return THROW_OR_RETURN_VALUE( + ClusterInfoImpl::create(params.server_context_.initManager(), params.server_context_, + params.cluster_, params.bind_config_, + params.server_context_.runtime(), std::move(socket_matcher), + std::move(scope), params.added_via_api_, factory_context), + std::unique_ptr); } void HdsCluster::initHealthchecks() { diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 6a99595c4574..2c5860e9321b 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1132,13 +1132,30 @@ LegacyLbPolicyConfigHelper::getTypedLbConfigFromLegacyProto( using ProtocolOptionsHashMap = absl::flat_hash_map; +absl::StatusOr> +ClusterInfoImpl::create(Init::Manager& info, + Server::Configuration::ServerFactoryContext& server_context, + const envoy::config::cluster::v3::Cluster& config, + const absl::optional& bind_config, + Runtime::Loader& runtime, TransportSocketMatcherPtr&& socket_matcher, + Stats::ScopeSharedPtr&& stats_scope, bool added_via_api, + Server::Configuration::TransportSocketFactoryContext& ctx) { + absl::Status creation_status = absl::OkStatus(); + auto ret = std::unique_ptr(new ClusterInfoImpl( + info, server_context, config, bind_config, runtime, std::move(socket_matcher), + std::move(stats_scope), added_via_api, ctx, creation_status)); + RETURN_IF_NOT_OK(creation_status); + return ret; +} + ClusterInfoImpl::ClusterInfoImpl( Init::Manager& init_manager, Server::Configuration::ServerFactoryContext& server_context, const envoy::config::cluster::v3::Cluster& config, const absl::optional& bind_config, Runtime::Loader& runtime, TransportSocketMatcherPtr&& socket_matcher, Stats::ScopeSharedPtr&& stats_scope, bool added_via_api, - Server::Configuration::TransportSocketFactoryContext& factory_context) + Server::Configuration::TransportSocketFactoryContext& factory_context, + absl::Status& creation_status) : runtime_(runtime), name_(config.name()), observability_name_(!config.alt_stat_name().empty() ? std::make_unique(config.alt_stat_name()) @@ -1229,8 +1246,17 @@ ClusterInfoImpl::ClusterInfoImpl( http_protocol_options_->common_http_protocol_options_, max_headers_count, runtime_.snapshot().getInteger(Http::MaxResponseHeadersCountOverrideKey, Http::DEFAULT_MAX_HEADERS_COUNT))), - max_response_headers_kb_(PROTOBUF_GET_OPTIONAL_WRAPPED( - http_protocol_options_->common_http_protocol_options_, max_response_headers_kb)), + max_response_headers_kb_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( + http_protocol_options_->common_http_protocol_options_, max_response_headers_kb, + [&]() -> absl::optional { + constexpr uint64_t unspecified = 0; + uint64_t runtime_val = runtime_.snapshot().getInteger( + Http::MaxResponseHeadersSizeOverrideKey, unspecified); + if (runtime_val == unspecified) { + return absl::nullopt; + } + return runtime_val; + }())), type_(config.type()), drain_connections_on_host_removal_(config.ignore_health_on_host_removal()), connection_pool_per_downstream_connection_( @@ -1244,9 +1270,10 @@ ClusterInfoImpl::ClusterInfoImpl( config.track_cluster_stats().per_endpoint_stats()) { #ifdef WIN32 if (set_local_interface_name_on_upstream_connections_) { - throwEnvoyExceptionOrPanic( + creation_status = absl::InvalidArgumentError( "set_local_interface_name_on_upstream_connections_ cannot be set to true " "on Windows platforms"); + return; } #endif @@ -1255,21 +1282,25 @@ ClusterInfoImpl::ClusterInfoImpl( // TODO(ggreenway): Verify that bypassing virtual dispatch here was intentional if (ClusterInfoImpl::perEndpointStatsEnabled() && server_context.bootstrap().cluster_manager().has_load_stats_config()) { - throwEnvoyExceptionOrPanic("Only one of cluster per_endpoint_stats and cluster manager " - "load_stats_config can be specified"); + creation_status = + absl::InvalidArgumentError("Only one of cluster per_endpoint_stats and cluster manager " + "load_stats_config can be specified"); + return; } if (config.has_max_requests_per_connection() && http_protocol_options_->common_http_protocol_options_.has_max_requests_per_connection()) { - throwEnvoyExceptionOrPanic("Only one of max_requests_per_connection from Cluster or " - "HttpProtocolOptions can be specified"); + creation_status = + absl::InvalidArgumentError("Only one of max_requests_per_connection from Cluster or " + "HttpProtocolOptions can be specified"); + return; } if (config.has_load_balancing_policy() || config.lb_policy() == envoy::config::cluster::v3::Cluster::LOAD_BALANCING_POLICY_CONFIG) { // If load_balancing_policy is set we will use it directly, ignoring lb_policy. - THROW_IF_NOT_OK(configureLbPolicies(config, server_context)); + SET_AND_RETURN_IF_NOT_OK(configureLbPolicies(config, server_context), creation_status); } else { // If load_balancing_policy is not set, we will try to convert legacy lb_policy // to load_balancing_policy and use it. @@ -1277,10 +1308,7 @@ ClusterInfoImpl::ClusterInfoImpl( auto lb_pair = LegacyLbPolicyConfigHelper::getTypedLbConfigFromLegacyProto( lb_factory_context, config, server_context.messageValidationVisitor()); - - if (!lb_pair.ok()) { - throwEnvoyExceptionOrPanic(std::string(lb_pair.status().message())); - } + SET_AND_RETURN_IF_NOT_OK(lb_pair.status(), creation_status); load_balancer_factory_ = lb_pair->factory; ASSERT(load_balancer_factory_ != nullptr, "null load balancer factory"); load_balancer_config_ = std::move(lb_pair->config); @@ -1288,9 +1316,11 @@ ClusterInfoImpl::ClusterInfoImpl( if (config.lb_subset_config().locality_weight_aware() && !config.common_lb_config().has_locality_weighted_lb_config()) { - throwEnvoyExceptionOrPanic(fmt::format("Locality weight aware subset LB requires that a " - "locality_weighted_lb_config be set in {}", - name_)); + creation_status = + absl::InvalidArgumentError(fmt::format("Locality weight aware subset LB requires that a " + "locality_weighted_lb_config be set in {}", + name_)); + return; } // Use default (1h) or configured `idle_timeout`, unless it's set to 0, indicating that no @@ -1336,7 +1366,8 @@ ClusterInfoImpl::ClusterInfoImpl( if (config.has_eds_cluster_config()) { if (config.type() != envoy::config::cluster::v3::Cluster::EDS) { - throwEnvoyExceptionOrPanic("eds_cluster_config set in a non-EDS cluster"); + creation_status = absl::InvalidArgumentError("eds_cluster_config set in a non-EDS cluster"); + return; } } @@ -1356,7 +1387,9 @@ ClusterInfoImpl::ClusterInfoImpl( if (proto_config.has_config_discovery()) { if (proto_config.has_typed_config()) { - throwEnvoyExceptionOrPanic("Only one of typed_config or config_discovery can be used"); + creation_status = + absl::InvalidArgumentError("Only one of typed_config or config_discovery can be used"); + return; } ENVOY_LOG(debug, " dynamic filter name: {}", proto_config.name()); @@ -1396,9 +1429,10 @@ ClusterInfoImpl::ClusterInfoImpl( const auto last_type_url = Config::Utility::getFactoryType(http_filters[http_filters.size() - 1].typed_config()); if (last_type_url != upstream_codec_type_url) { - throwEnvoyExceptionOrPanic(fmt::format( + creation_status = absl::InvalidArgumentError(fmt::format( "The codec filter is the only valid terminal upstream HTTP filter, use '{}'", upstream_codec_type_url)); + return; } } @@ -1407,8 +1441,9 @@ ClusterInfoImpl::ClusterInfoImpl( Server::Configuration::UpstreamHttpFilterConfigFactory> helper(*http_filter_config_provider_manager_, upstream_context_.serverFactoryContext(), factory_context.clusterManager(), upstream_context_, prefix); - THROW_IF_NOT_OK(helper.processFilters(http_filters, "upstream http", "upstream http", - http_filter_factories_)); + SET_AND_RETURN_IF_NOT_OK(helper.processFilters(http_filters, "upstream http", "upstream http", + http_filter_factories_), + creation_status); } } @@ -1586,12 +1621,13 @@ ClusterImplBase::ClusterImplBase(const envoy::config::cluster::v3::Cluster& clus auto socket_matcher = std::move(*socket_matcher_or_error); const bool matcher_supports_alpn = socket_matcher->allMatchesSupportAlpn(); auto& dispatcher = server_context.mainThreadDispatcher(); + auto info_or_error = ClusterInfoImpl::create( + init_manager_, server_context, cluster, cluster_context.clusterManager().bindConfig(), + runtime_, std::move(socket_matcher), std::move(stats_scope), cluster_context.addedViaApi(), + *transport_factory_context_); + SET_AND_RETURN_IF_NOT_OK(info_or_error.status(), creation_status); info_ = std::shared_ptr( - new ClusterInfoImpl(init_manager_, server_context, cluster, - cluster_context.clusterManager().bindConfig(), runtime_, - std::move(socket_matcher), std::move(stats_scope), - cluster_context.addedViaApi(), *transport_factory_context_), - [&dispatcher](const ClusterInfoImpl* self) { + (*info_or_error).release(), [&dispatcher](const ClusterInfoImpl* self) { ENVOY_LOG(trace, "Schedule destroy cluster info {}", self->name()); dispatcher.deleteInDispatcherThread( std::unique_ptr(self)); diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index b3c93e451bb7..6b537c75d9cd 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -812,12 +812,13 @@ class ClusterInfoImpl : public ClusterInfo, using HttpProtocolOptionsConfigImpl = Envoy::Extensions::Upstreams::Http::ProtocolOptionsConfigImpl; using TcpProtocolOptionsConfigImpl = Envoy::Extensions::Upstreams::Tcp::ProtocolOptionsConfigImpl; - ClusterInfoImpl(Init::Manager& info, Server::Configuration::ServerFactoryContext& server_context, - const envoy::config::cluster::v3::Cluster& config, - const absl::optional& bind_config, - Runtime::Loader& runtime, TransportSocketMatcherPtr&& socket_matcher, - Stats::ScopeSharedPtr&& stats_scope, bool added_via_api, - Server::Configuration::TransportSocketFactoryContext&); + static absl::StatusOr> + create(Init::Manager& info, Server::Configuration::ServerFactoryContext& server_context, + const envoy::config::cluster::v3::Cluster& config, + const absl::optional& bind_config, + Runtime::Loader& runtime, TransportSocketMatcherPtr&& socket_matcher, + Stats::ScopeSharedPtr&& stats_scope, bool added_via_api, + Server::Configuration::TransportSocketFactoryContext&); static DeferredCreationCompatibleClusterTrafficStats generateStats(Stats::ScopeSharedPtr scope, const ClusterTrafficStatNames& cluster_stat_names, @@ -1038,6 +1039,14 @@ class ClusterInfoImpl : public ClusterInfo, } protected: + ClusterInfoImpl(Init::Manager& info, Server::Configuration::ServerFactoryContext& server_context, + const envoy::config::cluster::v3::Cluster& config, + const absl::optional& bind_config, + Runtime::Loader& runtime, TransportSocketMatcherPtr&& socket_matcher, + Stats::ScopeSharedPtr&& stats_scope, bool added_via_api, + Server::Configuration::TransportSocketFactoryContext& context, + absl::Status& creation_status); + // Gets the retry budget percent/concurrency from the circuit breaker thresholds. If the retry // budget message is specified, defaults will be filled in if either params are unspecified. static std::pair, absl::optional> diff --git a/source/extensions/access_loggers/common/stream_access_log_common_impl.h b/source/extensions/access_loggers/common/stream_access_log_common_impl.h index 5b072331246e..6f7b3582ad37 100644 --- a/source/extensions/access_loggers/common/stream_access_log_common_impl.h +++ b/source/extensions/access_loggers/common/stream_access_log_common_impl.h @@ -23,7 +23,9 @@ createStreamAccessLogInstance(const Protobuf::Message& config, AccessLog::Filter Formatter::FormatterBasePtr); } else if (fal_config.access_log_format_case() == T::AccessLogFormatCase::ACCESS_LOG_FORMAT_NOT_SET) { - formatter = Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(); + formatter = THROW_OR_RETURN_VALUE( + Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + Formatter::FormatterPtr); } Filesystem::FilePathAndType file_info{destination_type, ""}; return std::make_shared( diff --git a/source/extensions/access_loggers/file/config.cc b/source/extensions/access_loggers/file/config.cc index 2761aa5064ce..33d4eefca2d3 100644 --- a/source/extensions/access_loggers/file/config.cc +++ b/source/extensions/access_loggers/file/config.cc @@ -31,7 +31,9 @@ FileAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, switch (fal_config.access_log_format_case()) { case envoy::extensions::access_loggers::file::v3::FileAccessLog::AccessLogFormatCase::kFormat: if (fal_config.format().empty()) { - formatter = Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(); + formatter = THROW_OR_RETURN_VALUE( + Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + Formatter::FormatterPtr); } else { envoy::config::core::v3::SubstitutionFormatString sff_config; sff_config.mutable_text_format_source()->set_inline_string(fal_config.format()); @@ -60,7 +62,9 @@ FileAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, break; case envoy::extensions::access_loggers::file::v3::FileAccessLog::AccessLogFormatCase:: ACCESS_LOG_FORMAT_NOT_SET: - formatter = Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(); + formatter = THROW_OR_RETURN_VALUE( + Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + Formatter::FormatterPtr); break; } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc index 2aab12ea6a61..ac0bcb9d0d19 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc @@ -275,14 +275,7 @@ void Utility::extractCommonAccessLogProperties( } if (stream_info.upstreamInfo().has_value()) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif const auto& upstream_info = stream_info.upstreamInfo().value().get(); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif if (upstream_info.upstreamHost() != nullptr) { Network::Utility::addressToProtobufAddress( *upstream_info.upstreamHost()->address(), diff --git a/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc b/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc index a99b80878dd2..796915cde9a5 100644 --- a/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc +++ b/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc @@ -78,7 +78,8 @@ OpenTelemetryFormatter::FormatBuilder::toFormatListValue( std::vector OpenTelemetryFormatter::FormatBuilder::toFormatStringValue(const std::string& string_format) const { - return Formatter::SubstitutionFormatParser::parse(string_format, commands_); + return THROW_OR_RETURN_VALUE(Formatter::SubstitutionFormatParser::parse(string_format, commands_), + std::vector); } ::opentelemetry::proto::common::v1::AnyValue OpenTelemetryFormatter::providersCallback( diff --git a/source/extensions/common/dubbo/hessian2_utils.h b/source/extensions/common/dubbo/hessian2_utils.h index 30a0f54335d6..a51a224bc836 100644 --- a/source/extensions/common/dubbo/hessian2_utils.h +++ b/source/extensions/common/dubbo/hessian2_utils.h @@ -5,15 +5,7 @@ #include "envoy/buffer/buffer.h" #include "absl/strings/string_view.h" - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif #include "hessian2/basic_codec/object_codec.hpp" -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif #include "hessian2/codec.hpp" #include "hessian2/object.hpp" #include "hessian2/reader.hpp" diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index d7841bad164c..fffd784255a6 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -66,6 +66,8 @@ DnsCacheImpl::DnsCacheImpl( ENVOY_LOG(debug, "DNS pre-resolve starting for host {}", host); startCacheLoad(host, hostname.port_value(), false, false); } + enable_dfp_dns_trace_ = context.serverFactoryContext().runtime().snapshot().getBoolean( + "envoy.enable_dfp_dns_trace", false); } DnsCacheImpl::~DnsCacheImpl() { @@ -256,11 +258,10 @@ DnsCacheImpl::PrimaryHostInfo& DnsCacheImpl::getPrimaryHost(const std::string& h void DnsCacheImpl::onResolveTimeout(const std::string& host) { ASSERT(main_thread_dispatcher_.isThreadSafe()); - auto& primary_host = getPrimaryHost(host); ENVOY_LOG_EVENT(debug, "dns_cache_resolve_timeout", "host='{}' resolution timeout", host); stats_.dns_query_timeout_.inc(); - primary_host.active_query_->cancel(Network::ActiveDnsQuery::CancelReason::Timeout); - finishResolve(host, Network::DnsResolver::ResolutionStatus::Failure, "resolve_timeout", {}); + finishResolve(host, Network::DnsResolver::ResolutionStatus::Failure, "resolve_timeout", {}, + absl::nullopt, /* is_proxy_lookup= */ false, /* is_timeout= */ true); } void DnsCacheImpl::onReResolveAlarm(const std::string& host) { @@ -379,7 +380,7 @@ void DnsCacheImpl::finishResolve(const std::string& host, absl::string_view details, std::list&& response, absl::optional resolution_time, - bool is_proxy_lookup) { + bool is_proxy_lookup, bool is_timeout) { ASSERT(main_thread_dispatcher_.isThreadSafe()); if (Runtime::runtimeFeatureEnabled( "envoy.reloadable_features.dns_cache_set_ip_version_to_remove")) { @@ -417,6 +418,18 @@ void DnsCacheImpl::finishResolve(const std::string& host, return primary_host_it->second.get(); }(); + std::string details_with_maybe_trace = std::string(details); + if (primary_host_info != nullptr && primary_host_info->active_query_ != nullptr) { + if (enable_dfp_dns_trace_) { + std::string traces = primary_host_info->active_query_->getTraces(); + details_with_maybe_trace = absl::StrCat(details, ":", traces); + } + // `cancel` must be called last because the `ActiveQuery` will be destroyed afterward. + if (is_timeout) { + primary_host_info->active_query_->cancel(Network::ActiveDnsQuery::CancelReason::Timeout); + } + } + bool first_resolve = false; if (!from_cache) { @@ -480,7 +493,7 @@ void DnsCacheImpl::finishResolve(const std::string& host, current_address ? current_address->asStringView() : "", new_address ? new_address->asStringView() : ""); primary_host_info->host_info_->setAddresses(new_address, std::move(address_list)); - primary_host_info->host_info_->setDetails(std::string(details)); + primary_host_info->host_info_->setDetails(details_with_maybe_trace); runAddUpdateCallbacks(host, primary_host_info->host_info_); primary_host_info->host_info_->setFirstResolveComplete(); @@ -490,7 +503,7 @@ void DnsCacheImpl::finishResolve(const std::string& host, // We only set details here if current address is null because but // non-null->null resolutions we don't update the address so will use a // previously resolved address + details. - primary_host_info->host_info_->setDetails(std::string(details)); + primary_host_info->host_info_->setDetails(details_with_maybe_trace); } if (first_resolve) { diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index fe8d6819447d..15e0b41408d7 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -145,7 +145,7 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable&& response, absl::optional resolution_time = {}, - bool is_proxy_lookup = false); + bool is_proxy_lookup = false, bool is_timeout = false); void runAddUpdateCallbacks(const std::string& host, const DnsHostInfoSharedPtr& host_info); void runResolutionCompleteCallbacks(const std::string& host, const DnsHostInfoSharedPtr& host_info, @@ -270,6 +270,7 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable ip_version_to_remove_ ABSL_GUARDED_BY(ip_version_to_remove_lock_) = absl::nullopt; + bool enable_dfp_dns_trace_; }; } // namespace DynamicForwardProxy diff --git a/source/extensions/config_subscription/grpc/delta_subscription_state.cc b/source/extensions/config_subscription/grpc/delta_subscription_state.cc index 7a1f2fd35445..1d1b949323c6 100644 --- a/source/extensions/config_subscription/grpc/delta_subscription_state.cc +++ b/source/extensions/config_subscription/grpc/delta_subscription_state.cc @@ -218,12 +218,14 @@ void DeltaSubscriptionState::handleGoodResponse( } } - watch_map_.onConfigUpdate(non_heartbeat_resources, message.removed_resources(), + absl::Span non_heartbeat_resources_span = + absl::MakeConstSpan(non_heartbeat_resources.data(), non_heartbeat_resources.size()); + watch_map_.onConfigUpdate(non_heartbeat_resources_span, message.removed_resources(), message.system_version_info()); // Processing point when resources are successfully ingested. if (xds_config_tracker_.has_value()) { - xds_config_tracker_->onConfigAccepted(message.type_url(), non_heartbeat_resources, + xds_config_tracker_->onConfigAccepted(message.type_url(), non_heartbeat_resources_span, message.removed_resources()); } diff --git a/source/extensions/config_subscription/grpc/grpc_mux_failover.h b/source/extensions/config_subscription/grpc/grpc_mux_failover.h index 19765df61fc7..bea808a5212f 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_failover.h +++ b/source/extensions/config_subscription/grpc/grpc_mux_failover.h @@ -102,6 +102,25 @@ class GrpcMuxFailover : public GrpcStreamInterface, "Already connected to an xDS server, skipping establishNewStream() call"); return; } + if (!Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.xds_failover_to_primary_enabled")) { + // Allow stickiness, so if Envoy was ever connected to the primary source only + // retry to reconnect to the primary source, If Envoy was ever connected to the + // failover source then only retry to reconnect to the failover source. + if (previously_connected_to_ == ConnectedTo::Primary) { + ENVOY_LOG_MISC( + info, "Previously connected to the primary xDS source, attempting to reconnect to it"); + connection_state_ = ConnectionState::ConnectingToPrimary; + primary_grpc_stream_->establishNewStream(); + return; + } else if (previously_connected_to_ == ConnectedTo::Failover) { + ENVOY_LOG_MISC( + info, "Previously connected to the failover xDS source, attempting to reconnect to it"); + connection_state_ = ConnectionState::ConnectingToFailover; + failover_grpc_stream_->establishNewStream(); + return; + } + } // connection_state_ is either None, ConnectingToPrimary or // ConnectingToFailover. In the first 2 cases try to connect to the primary // (preferring the primary in the case of None), and in the third case @@ -188,6 +207,8 @@ class GrpcMuxFailover : public GrpcStreamInterface, } private: + friend class GrpcMuxFailoverTest; + // A helper class that proxies the callbacks of GrpcStreamCallbacks for the primary service. class PrimaryGrpcStreamCallbacks : public GrpcStreamCallbacks { public: @@ -287,10 +308,30 @@ class GrpcMuxFailover : public GrpcStreamInterface, // This will be called when the failover stream fails to establish a connection, or after the // connection was closed. ASSERT(parent_.connectingToOrConnectedToFailover()); + if (!Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.xds_failover_to_primary_enabled")) { + // If previously Envoy was connected to the failover, keep using that. + // Otherwise let the retry mechanism try to access the primary (similar + // to if the runtime flag was not set). + if (parent_.previously_connected_to_ == ConnectedTo::Failover) { + ENVOY_LOG(debug, + "Failover xDS stream disconnected (either after establishing a connection or " + "before). Attempting to reconnect to Failover because Envoy successfully " + "connected to it previously."); + // Not closing the failover stream, allows it to use its retry timer + // to reconnect to the failover source. + // Next attempt will be to the failover after Envoy was already + // connected to it. Allowing to send the initial_resource_versions on reconnect. + parent_.grpc_mux_callbacks_.onEstablishmentFailure(true); + parent_.connection_state_ = ConnectionState::ConnectingToFailover; + return; + } + } // Either this was an intentional disconnection from the failover source, // or unintentional. Either way, try to connect to the primary next. - ENVOY_LOG(debug, "Failover xDS stream diconnected (either after establishing a connection or " - "before). Attempting to connect to the primary stream."); + ENVOY_LOG(debug, + "Failover xDS stream disconnected (either after establishing a connection or " + "before). Attempting to connect to the primary stream."); // This will close the stream and prevent the retry timer from // reconnecting to the failover source. The next attempt will be to the @@ -356,7 +397,15 @@ class GrpcMuxFailover : public GrpcStreamInterface, void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override { PANIC("not implemented"); } - void closeStream() override { PANIC("not implemented"); } + void closeStream() override { + if (connectingToOrConnectedToPrimary()) { + ENVOY_LOG_MISC(debug, "Intentionally closing the primary gRPC stream"); + primary_grpc_stream_->closeStream(); + } else if (connectingToOrConnectedToFailover()) { + ENVOY_LOG_MISC(debug, "Intentionally closing the failover gRPC stream"); + failover_grpc_stream_->closeStream(); + } + } // The stream callbacks that will be invoked on the GrpcMux object, to notify // about the state of the underlying primary/failover stream. @@ -398,8 +447,8 @@ class GrpcMuxFailover : public GrpcStreamInterface, // exclusive), it can attempt connecting to more than one source at a time. ConnectionState connection_state_; - // A flag that keeps track of whether Envoy successfully connected to either the - // primary or failover source. Envoy is considered successfully connected to a source + // A flag that keeps track of whether Envoy successfully connected to the + // primary source. Envoy is considered successfully connected to a source // once it receives a response from it. bool ever_connected_to_primary_{false}; diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc index 4317cdca1a2c..f036cea34e64 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc @@ -60,14 +60,18 @@ std::string convertToWildcard(const std::string& resource_name) { } // namespace GrpcMuxImpl::GrpcMuxImpl(GrpcMuxContext& grpc_mux_context, bool skip_subsequent_node) - : grpc_stream_(createGrpcStreamObject(grpc_mux_context)), + : dispatcher_(grpc_mux_context.dispatcher_), + grpc_stream_(createGrpcStreamObject(std::move(grpc_mux_context.async_client_), + std::move(grpc_mux_context.failover_async_client_), + grpc_mux_context.service_method_, grpc_mux_context.scope_, + std::move(grpc_mux_context.backoff_strategy_), + grpc_mux_context.rate_limit_settings_)), local_info_(grpc_mux_context.local_info_), skip_subsequent_node_(skip_subsequent_node), config_validators_(std::move(grpc_mux_context.config_validators_)), xds_config_tracker_(grpc_mux_context.xds_config_tracker_), xds_resources_delegate_(grpc_mux_context.xds_resources_delegate_), eds_resources_cache_(std::move(grpc_mux_context.eds_resources_cache_)), target_xds_authority_(grpc_mux_context.target_xds_authority_), - dispatcher_(grpc_mux_context.dispatcher_), dynamic_update_callback_handle_( grpc_mux_context.local_info_.contextProvider().addDynamicContextUpdateCallback( [this](absl::string_view resource_type_url) { @@ -80,29 +84,33 @@ GrpcMuxImpl::GrpcMuxImpl(GrpcMuxContext& grpc_mux_context, bool skip_subsequent_ std::unique_ptr> -GrpcMuxImpl::createGrpcStreamObject(GrpcMuxContext& grpc_mux_context) { +GrpcMuxImpl::createGrpcStreamObject(Grpc::RawAsyncClientPtr&& async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + const Protobuf::MethodDescriptor& service_method, + Stats::Scope& scope, BackOffStrategyPtr&& backoff_strategy, + const RateLimitSettings& rate_limit_settings) { if (Runtime::runtimeFeatureEnabled("envoy.restart_features.xds_failover_support")) { return std::make_unique>( /*primary_stream_creator=*/ - [&grpc_mux_context]( + [&async_client, &service_method, &dispatcher = dispatcher_, &scope, &backoff_strategy, + &rate_limit_settings]( GrpcStreamCallbacks* callbacks) -> GrpcStreamInterfacePtr { return std::make_unique>( - callbacks, std::move(grpc_mux_context.async_client_), - grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, - grpc_mux_context.scope_, std::move(grpc_mux_context.backoff_strategy_), - grpc_mux_context.rate_limit_settings_, + callbacks, std::move(async_client), service_method, dispatcher, scope, + std::move(backoff_strategy), rate_limit_settings, GrpcStream::ConnectedStateValue:: FIRST_ENTRY); }, /*failover_stream_creator=*/ - grpc_mux_context.failover_async_client_ + failover_async_client ? absl::make_optional( - [&grpc_mux_context]( + [&failover_async_client, &service_method, &dispatcher = dispatcher_, &scope, + &rate_limit_settings]( GrpcStreamCallbacks* callbacks) -> GrpcStreamInterfacePtr>( - callbacks, std::move(grpc_mux_context.failover_async_client_), - grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, - grpc_mux_context.scope_, + callbacks, std::move(failover_async_client), service_method, dispatcher, + scope, // TODO(adisuissa): the backoff strategy for the failover should // be the same as the primary source. std::make_unique( GrpcMuxFailover:: DefaultFailoverBackoffMilliseconds), - grpc_mux_context.rate_limit_settings_, + rate_limit_settings, GrpcStream:: ConnectedStateValue::SECOND_ENTRY); }) : absl::nullopt, /*grpc_mux_callbacks=*/*this, - /*dispatch=*/grpc_mux_context.dispatcher_); + /*dispatch=*/dispatcher_); } return std::make_unique>( - this, std::move(grpc_mux_context.async_client_), grpc_mux_context.service_method_, - grpc_mux_context.dispatcher_, grpc_mux_context.scope_, - std::move(grpc_mux_context.backoff_strategy_), grpc_mux_context.rate_limit_settings_, + this, std::move(async_client), service_method, dispatcher_, scope, + std::move(backoff_strategy), rate_limit_settings, GrpcStream< envoy::service::discovery::v3::DiscoveryRequest, envoy::service::discovery::v3::DiscoveryResponse>::ConnectedStateValue::FIRST_ENTRY); @@ -292,6 +298,37 @@ GrpcMuxWatchPtr GrpcMuxImpl::addWatch(const std::string& type_url, return watch; } +absl::Status +GrpcMuxImpl::updateMuxSource(Grpc::RawAsyncClientPtr&& primary_async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + CustomConfigValidatorsPtr&& custom_config_validators, + Stats::Scope& scope, BackOffStrategyPtr&& backoff_strategy, + const envoy::config::core::v3::ApiConfigSource& ads_config_source) { + // Process the rate limit settings. + absl::StatusOr rate_limit_settings_or_error = + Utility::parseRateLimitSettings(ads_config_source); + RETURN_IF_NOT_OK_REF(rate_limit_settings_or_error.status()); + + const Protobuf::MethodDescriptor& service_method = + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v3.AggregatedDiscoveryService.StreamAggregatedResources"); + + // Disconnect from current xDS servers. + ENVOY_LOG_MISC(info, "Replacing xDS gRPC mux source"); + grpc_stream_->closeStream(); + grpc_stream_ = createGrpcStreamObject(std::move(primary_async_client), + std::move(failover_async_client), service_method, scope, + std::move(backoff_strategy), *rate_limit_settings_or_error); + + // Update the config validators. + config_validators_ = std::move(custom_config_validators); + + // Start the subscriptions over the grpc_stream. + grpc_stream_->establishNewStream(); + + return absl::OkStatus(); +} + ScopedResume GrpcMuxImpl::pause(const std::string& type_url) { return pause(std::vector{type_url}); } diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/grpc_mux_impl.h index 885942f3fb2a..c53d51cd0979 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.h @@ -73,6 +73,13 @@ class GrpcMuxImpl : public GrpcMux, return makeOptRefFromPtr(eds_resources_cache_.get()); } + absl::Status + updateMuxSource(Grpc::RawAsyncClientPtr&& primary_async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + CustomConfigValidatorsPtr&& custom_config_validators, Stats::Scope& scope, + BackOffStrategyPtr&& backoff_strategy, + const envoy::config::core::v3::ApiConfigSource& ads_config_source) override; + void handleDiscoveryResponse( std::unique_ptr&& message); @@ -100,11 +107,13 @@ class GrpcMuxImpl : public GrpcMux, private: // Helper function to create the grpc_stream_ object. - // TODO(adisuissa): this should be removed when envoy.restart_features.xds_failover_support - // is deprecated. std::unique_ptr> - createGrpcStreamObject(GrpcMuxContext& grpc_mux_context); + createGrpcStreamObject(Grpc::RawAsyncClientPtr&& async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, + BackOffStrategyPtr&& backoff_strategy, + const RateLimitSettings& rate_limit_settings); void drainRequests(); void setRetryTimer(); @@ -272,6 +281,7 @@ class GrpcMuxImpl : public GrpcMux, ApiState& api_state, const std::string& type_url, const std::string& version_info, bool call_delegate); + Event::Dispatcher& dispatcher_; // Multiplexes the stream to the primary and failover sources. // TODO(adisuissa): Once envoy.restart_features.xds_failover_support is deprecated, // convert from unique_ptr to GrpcMuxFailover directly. @@ -301,7 +311,6 @@ class GrpcMuxImpl : public GrpcMux, // URL. std::unique_ptr> request_queue_; - Event::Dispatcher& dispatcher_; Common::CallbackHandlePtr dynamic_update_callback_handle_; bool started_{false}; diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc index 1a47a63572c7..6315853829ef 100644 --- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc @@ -36,7 +36,12 @@ using AllMuxes = ThreadSafeSingleton; } // namespace NewGrpcMuxImpl::NewGrpcMuxImpl(GrpcMuxContext& grpc_mux_context) - : grpc_stream_(createGrpcStreamObject(grpc_mux_context)), + : dispatcher_(grpc_mux_context.dispatcher_), + grpc_stream_(createGrpcStreamObject(std::move(grpc_mux_context.async_client_), + std::move(grpc_mux_context.failover_async_client_), + grpc_mux_context.service_method_, grpc_mux_context.scope_, + std::move(grpc_mux_context.backoff_strategy_), + grpc_mux_context.rate_limit_settings_)), local_info_(grpc_mux_context.local_info_), config_validators_(std::move(grpc_mux_context.config_validators_)), dynamic_update_callback_handle_( @@ -45,7 +50,6 @@ NewGrpcMuxImpl::NewGrpcMuxImpl(GrpcMuxContext& grpc_mux_context) onDynamicContextUpdate(resource_type_url); return absl::OkStatus(); })), - dispatcher_(grpc_mux_context.dispatcher_), xds_config_tracker_(grpc_mux_context.xds_config_tracker_), eds_resources_cache_(std::move(grpc_mux_context.eds_resources_cache_)) { AllMuxes::get().insert(this); @@ -53,30 +57,34 @@ NewGrpcMuxImpl::NewGrpcMuxImpl(GrpcMuxContext& grpc_mux_context) std::unique_ptr> -NewGrpcMuxImpl::createGrpcStreamObject(GrpcMuxContext& grpc_mux_context) { +NewGrpcMuxImpl::createGrpcStreamObject(Grpc::RawAsyncClientPtr&& async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + const Protobuf::MethodDescriptor& service_method, + Stats::Scope& scope, BackOffStrategyPtr&& backoff_strategy, + const RateLimitSettings& rate_limit_settings) { if (Runtime::runtimeFeatureEnabled("envoy.restart_features.xds_failover_support")) { return std::make_unique>( /*primary_stream_creator=*/ - [&grpc_mux_context]( + [&async_client, &service_method, &dispatcher = dispatcher_, &scope, &backoff_strategy, + &rate_limit_settings]( GrpcStreamCallbacks* callbacks) -> GrpcStreamInterfacePtr { return std::make_unique< GrpcStream>( - callbacks, std::move(grpc_mux_context.async_client_), - grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, - grpc_mux_context.scope_, std::move(grpc_mux_context.backoff_strategy_), - grpc_mux_context.rate_limit_settings_, + callbacks, std::move(async_client), service_method, dispatcher, scope, + std::move(backoff_strategy), rate_limit_settings, GrpcStream:: ConnectedStateValue::FIRST_ENTRY); }, /*failover_stream_creator=*/ - grpc_mux_context.failover_async_client_ + failover_async_client ? absl::make_optional( - [&grpc_mux_context]( + [&failover_async_client, &service_method, &dispatcher = dispatcher_, &scope, + &rate_limit_settings]( GrpcStreamCallbacks* callbacks) -> GrpcStreamInterfacePtr< @@ -85,29 +93,27 @@ NewGrpcMuxImpl::createGrpcStreamObject(GrpcMuxContext& grpc_mux_context) { return std::make_unique< GrpcStream>( - callbacks, std::move(grpc_mux_context.failover_async_client_), - grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, - grpc_mux_context.scope_, + callbacks, std::move(failover_async_client), service_method, dispatcher, + scope, // TODO(adisuissa): the backoff strategy for the failover should // be the same as the primary source. std::make_unique( GrpcMuxFailover:: DefaultFailoverBackoffMilliseconds), - grpc_mux_context.rate_limit_settings_, + rate_limit_settings, GrpcStream:: ConnectedStateValue::SECOND_ENTRY); }) : absl::nullopt, /*grpc_mux_callbacks=*/*this, - /*dispatch=*/grpc_mux_context.dispatcher_); + /*dispatch=*/dispatcher_); } return std::make_unique>( - this, std::move(grpc_mux_context.async_client_), grpc_mux_context.service_method_, - grpc_mux_context.dispatcher_, grpc_mux_context.scope_, - std::move(grpc_mux_context.backoff_strategy_), grpc_mux_context.rate_limit_settings_, + this, std::move(async_client), service_method, dispatcher_, scope, + std::move(backoff_strategy), rate_limit_settings, GrpcStream< envoy::service::discovery::v3::DeltaDiscoveryRequest, envoy::service::discovery::v3::DeltaDiscoveryResponse>::ConnectedStateValue::FIRST_ENTRY); @@ -246,6 +252,41 @@ GrpcMuxWatchPtr NewGrpcMuxImpl::addWatch(const std::string& type_url, return std::make_unique(type_url, watch, *this, options); } +absl::Status +NewGrpcMuxImpl::updateMuxSource(Grpc::RawAsyncClientPtr&& primary_async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + CustomConfigValidatorsPtr&& custom_config_validators, + Stats::Scope& scope, BackOffStrategyPtr&& backoff_strategy, + const envoy::config::core::v3::ApiConfigSource& ads_config_source) { + // Process the rate limit settings. + absl::StatusOr rate_limit_settings_or_error = + Utility::parseRateLimitSettings(ads_config_source); + RETURN_IF_NOT_OK_REF(rate_limit_settings_or_error.status()); + + const Protobuf::MethodDescriptor& service_method = + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v3.AggregatedDiscoveryService.DeltaAggregatedResources"); + + // Disconnect from current xDS servers. + ENVOY_LOG_MISC(info, "Replacing the xDS gRPC mux source"); + grpc_stream_->closeStream(); + grpc_stream_ = createGrpcStreamObject(std::move(primary_async_client), + std::move(failover_async_client), service_method, scope, + std::move(backoff_strategy), *rate_limit_settings_or_error); + + // Update the config validators. + config_validators_ = std::move(custom_config_validators); + // Update the watch map's config validators. + for (auto& [type_url, subscription] : subscriptions_) { + subscription->watch_map_.setConfigValidators(config_validators_.get()); + } + + // Start the subscriptions over the grpc_stream. + grpc_stream_->establishNewStream(); + + return absl::OkStatus(); +} + // Updates the list of resource names watched by the given watch. If an added name is new across // the whole subscription, or if a removed name has no other watch interested in it, then the // subscription will enqueue and attempt to send an appropriate discovery request. @@ -320,7 +361,7 @@ void NewGrpcMuxImpl::addSubscription(const std::string& type_url, } subscriptions_.emplace( type_url, std::make_unique(type_url, local_info_, use_namespace_matching, - dispatcher_, *config_validators_.get(), + dispatcher_, config_validators_.get(), xds_config_tracker_, resources_cache)); subscription_ordering_.emplace_back(type_url); } diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h index 741ea6856e45..5c35dd6e177b 100644 --- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h @@ -78,6 +78,13 @@ class NewGrpcMuxImpl // TODO(fredlas) remove this from the GrpcMux interface. void start() override; + absl::Status + updateMuxSource(Grpc::RawAsyncClientPtr&& primary_async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + CustomConfigValidatorsPtr&& custom_config_validators, Stats::Scope& scope, + BackOffStrategyPtr&& backoff_strategy, + const envoy::config::core::v3::ApiConfigSource& ads_config_source) override; + GrpcStreamInterface& grpcStreamForTest() { @@ -95,7 +102,7 @@ class NewGrpcMuxImpl struct SubscriptionStuff { SubscriptionStuff(const std::string& type_url, const LocalInfo::LocalInfo& local_info, const bool use_namespace_matching, Event::Dispatcher& dispatcher, - CustomConfigValidators& config_validators, + CustomConfigValidators* config_validators, XdsConfigTrackerOptRef xds_config_tracker, EdsResourcesCacheOptRef eds_resources_cache) : watch_map_(use_namespace_matching, type_url, config_validators, eds_resources_cache), @@ -149,11 +156,13 @@ class NewGrpcMuxImpl }; // Helper function to create the grpc_stream_ object. - // TODO(adisuissa): this should be removed when envoy.restart_features.xds_failover_support - // is deprecated. std::unique_ptr> - createGrpcStreamObject(GrpcMuxContext& grpc_mux_context); + createGrpcStreamObject(Grpc::RawAsyncClientPtr&& async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, + BackOffStrategyPtr&& backoff_strategy, + const RateLimitSettings& rate_limit_settings); void removeWatch(const std::string& type_url, Watch* watch); @@ -196,6 +205,7 @@ class NewGrpcMuxImpl // the order of Envoy's dependency ordering). std::list subscription_ordering_; + Event::Dispatcher& dispatcher_; // Multiplexes the stream to the primary and failover sources. // TODO(adisuissa): Once envoy.restart_features.xds_failover_support is deprecated, // convert from unique_ptr to GrpcMuxFailover directly. @@ -206,7 +216,6 @@ class NewGrpcMuxImpl const LocalInfo::LocalInfo& local_info_; CustomConfigValidatorsPtr config_validators_; Common::CallbackHandlePtr dynamic_update_callback_handle_; - Event::Dispatcher& dispatcher_; XdsConfigTrackerOptRef xds_config_tracker_; EdsResourcesCachePtr eds_resources_cache_; diff --git a/source/extensions/config_subscription/grpc/watch_map.cc b/source/extensions/config_subscription/grpc/watch_map.cc index 815bcca48581..ad6af7e18c1e 100644 --- a/source/extensions/config_subscription/grpc/watch_map.cc +++ b/source/extensions/config_subscription/grpc/watch_map.cc @@ -154,7 +154,7 @@ void WatchMap::onConfigUpdate(const std::vector& resources, } // Execute external config validators. - config_validators_.executeValidators(type_url_, resources); + config_validators_->executeValidators(type_url_, resources); const bool map_is_single_wildcard = (watches_.size() == 1 && wildcard_watches_.size() == 1); // We just bundled up the updates into nice per-watch packages. Now, deliver them. @@ -216,7 +216,7 @@ void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField } void WatchMap::onConfigUpdate( - const Protobuf::RepeatedPtrField& added_resources, + absl::Span added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { // Track any removals triggered by earlier watch updates. @@ -232,15 +232,15 @@ void WatchMap::onConfigUpdate( // resources the watch map is interested in. Reserve the correct amount of // space for the vector for the good case. decoded_resources.reserve(added_resources.size()); - for (const auto& r : added_resources) { - const absl::flat_hash_set& interested_in_r = watchesInterestedIn(r.name()); + for (const auto* r : added_resources) { + const absl::flat_hash_set& interested_in_r = watchesInterestedIn(r->name()); // If there are no watches, then we don't need to decode. If there are watches, they should all // be for the same resource type, so we can just use the callbacks of the first watch to decode. if (interested_in_r.empty()) { continue; } decoded_resources.emplace_back( - new DecodedResourceImpl((*interested_in_r.begin())->resource_decoder_, r)); + new DecodedResourceImpl((*interested_in_r.begin())->resource_decoder_, *r)); for (const auto& interested_watch : interested_in_r) { per_watch_added[interested_watch].emplace_back(*decoded_resources.back()); } @@ -254,7 +254,7 @@ void WatchMap::onConfigUpdate( } // Execute external config validators. - config_validators_.executeValidators(type_url_, decoded_resources, removed_resources); + config_validators_->executeValidators(type_url_, decoded_resources, removed_resources); // We just bundled up the updates into nice per-watch packages. Now, deliver them. for (const auto& [cur_watch, resource_to_add] : per_watch_added) { diff --git a/source/extensions/config_subscription/grpc/watch_map.h b/source/extensions/config_subscription/grpc/watch_map.h index 47cd43b4581f..1f824900c211 100644 --- a/source/extensions/config_subscription/grpc/watch_map.h +++ b/source/extensions/config_subscription/grpc/watch_map.h @@ -73,7 +73,7 @@ struct Watch { class WatchMap : public UntypedConfigUpdateCallbacks, public Logger::Loggable { public: WatchMap(const bool use_namespace_matching, const std::string& type_url, - CustomConfigValidators& config_validators, EdsResourcesCacheOptRef eds_resources_cache) + CustomConfigValidators* config_validators, EdsResourcesCacheOptRef eds_resources_cache) : use_namespace_matching_(use_namespace_matching), type_url_(type_url), config_validators_(config_validators), eds_resources_cache_(eds_resources_cache) { // If eds resources cache is provided, then the type must be ClusterLoadAssignment. @@ -105,15 +105,19 @@ class WatchMap : public UntypedConfigUpdateCallbacks, public Logger::Loggable& resources, const std::string& version_info) override; - void onConfigUpdate( - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override; + void + onConfigUpdate(absl::Span added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override; void onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) override; WatchMap(const WatchMap&) = delete; WatchMap& operator=(const WatchMap&) = delete; + void setConfigValidators(CustomConfigValidators* config_validators) { + config_validators_ = config_validators; + } + private: void removeDeferredWatches(); @@ -147,7 +151,7 @@ class WatchMap : public UntypedConfigUpdateCallbacks, public Logger::Loggable non_heartbeat_resources_span = + absl::MakeConstSpan(non_heartbeat_resources.data(), non_heartbeat_resources.size()); + callbacks().onConfigUpdate(non_heartbeat_resources_span, message.removed_resources(), message.system_version_info()); // Processing point when resources are successfully ingested. if (xds_config_tracker_.has_value()) { - xds_config_tracker_->onConfigAccepted(message.type_url(), non_heartbeat_resources, + xds_config_tracker_->onConfigAccepted(message.type_url(), non_heartbeat_resources_span, message.removed_resources()); } diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc index 273fb0081040..c418a9f265c8 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc @@ -40,7 +40,12 @@ using AllMuxes = ThreadSafeSingleton; template GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory, GrpcMuxContext& grpc_mux_context, bool skip_subsequent_node) - : grpc_stream_(createGrpcStreamObject(grpc_mux_context)), + : dispatcher_(grpc_mux_context.dispatcher_), + grpc_stream_(createGrpcStreamObject(std::move(grpc_mux_context.async_client_), + std::move(grpc_mux_context.failover_async_client_), + grpc_mux_context.service_method_, grpc_mux_context.scope_, + std::move(grpc_mux_context.backoff_strategy_), + grpc_mux_context.rate_limit_settings_)), subscription_state_factory_(std::move(subscription_state_factory)), skip_subsequent_node_(skip_subsequent_node), local_info_(grpc_mux_context.local_info_), dynamic_update_callback_handle_( @@ -59,44 +64,46 @@ GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_fac } template -std::unique_ptr> -GrpcMuxImpl::createGrpcStreamObject(GrpcMuxContext& grpc_mux_context) { +std::unique_ptr> GrpcMuxImpl::createGrpcStreamObject( + Grpc::RawAsyncClientPtr&& async_client, Grpc::RawAsyncClientPtr&& failover_async_client, + const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, + BackOffStrategyPtr&& backoff_strategy, const RateLimitSettings& rate_limit_settings) { if (Runtime::runtimeFeatureEnabled("envoy.restart_features.xds_failover_support")) { return std::make_unique>( /*primary_stream_creator=*/ - [&grpc_mux_context](GrpcStreamCallbacks* callbacks) -> GrpcStreamInterfacePtr { + [&async_client, &service_method, &dispatcher = dispatcher_, &scope, &backoff_strategy, + &rate_limit_settings]( + GrpcStreamCallbacks* callbacks) -> GrpcStreamInterfacePtr { return std::make_unique>( - callbacks, std::move(grpc_mux_context.async_client_), - grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, - grpc_mux_context.scope_, std::move(grpc_mux_context.backoff_strategy_), - grpc_mux_context.rate_limit_settings_, + callbacks, std::move(async_client), service_method, dispatcher, scope, + std::move(backoff_strategy), rate_limit_settings, GrpcStream::ConnectedStateValue::FIRST_ENTRY); }, /*failover_stream_creator=*/ - grpc_mux_context.failover_async_client_ - ? absl::make_optional([&grpc_mux_context](GrpcStreamCallbacks* callbacks) - -> GrpcStreamInterfacePtr { - return std::make_unique>( - callbacks, std::move(grpc_mux_context.failover_async_client_), - grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, - grpc_mux_context.scope_, - // TODO(adisuissa): the backoff strategy for the failover should - // be the same as the primary source. - std::make_unique( - GrpcMuxFailover::DefaultFailoverBackoffMilliseconds), - grpc_mux_context.rate_limit_settings_, - GrpcStream::ConnectedStateValue::SECOND_ENTRY); - }) + failover_async_client + ? absl::make_optional( + [&failover_async_client, &service_method, &dispatcher = dispatcher_, &scope, + &rate_limit_settings]( + GrpcStreamCallbacks* callbacks) -> GrpcStreamInterfacePtr { + return std::make_unique>( + callbacks, std::move(failover_async_client), service_method, dispatcher, + scope, + // TODO(adisuissa): the backoff strategy for the failover should + // be the same as the primary source. + std::make_unique( + GrpcMuxFailover::DefaultFailoverBackoffMilliseconds), + rate_limit_settings, GrpcStream::ConnectedStateValue::SECOND_ENTRY); + }) : absl::nullopt, /*grpc_mux_callbacks=*/*this, - /*dispatch=*/grpc_mux_context.dispatcher_); + /*dispatch=*/dispatcher_); } - return std::make_unique>( - this, std::move(grpc_mux_context.async_client_), grpc_mux_context.service_method_, - grpc_mux_context.dispatcher_, grpc_mux_context.scope_, - std::move(grpc_mux_context.backoff_strategy_), grpc_mux_context.rate_limit_settings_, - GrpcStream::ConnectedStateValue::FIRST_ENTRY); + return std::make_unique>(this, std::move(async_client), service_method, + dispatcher_, scope, std::move(backoff_strategy), + rate_limit_settings, + GrpcStream::ConnectedStateValue::FIRST_ENTRY); } + template GrpcMuxImpl::~GrpcMuxImpl() { AllMuxes::get().erase(this); } @@ -134,7 +141,7 @@ Config::GrpcMuxWatchPtr GrpcMuxImpl::addWatch( watch_map = watch_maps_ .emplace(type_url, std::make_unique(options.use_namespace_matching_, type_url, - *config_validators_.get(), resources_cache)) + config_validators_.get(), resources_cache)) .first; subscriptions_.emplace(type_url, subscription_state_factory_->makeSubscriptionState( type_url, *watch_maps_[type_url], resource_decoder, @@ -221,6 +228,40 @@ ScopedResume GrpcMuxImpl::pause(const std::vector typ }); } +template +absl::Status GrpcMuxImpl::updateMuxSource( + Grpc::RawAsyncClientPtr&& primary_async_client, Grpc::RawAsyncClientPtr&& failover_async_client, + CustomConfigValidatorsPtr&& custom_config_validators, Stats::Scope& scope, + BackOffStrategyPtr&& backoff_strategy, + const envoy::config::core::v3::ApiConfigSource& ads_config_source) { + // Process the rate limit settings. + absl::StatusOr rate_limit_settings_or_error = + Utility::parseRateLimitSettings(ads_config_source); + RETURN_IF_NOT_OK_REF(rate_limit_settings_or_error.status()); + + const Protobuf::MethodDescriptor& service_method = + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(methodName()); + + // Disconnect from current xDS servers. + ENVOY_LOG_MISC(info, "Replacing the xDS gRPC mux source"); + grpc_stream_->closeStream(); + grpc_stream_ = createGrpcStreamObject(std::move(primary_async_client), + std::move(failover_async_client), service_method, scope, + std::move(backoff_strategy), *rate_limit_settings_or_error); + + // Update the config validators. + config_validators_ = std::move(custom_config_validators); + // Update the watch map's config validators. + for (auto& [type_url, watch_map] : watch_maps_) { + watch_map->setConfigValidators(config_validators_.get()); + } + + // Start the subscriptions over the grpc_stream. + grpc_stream_->establishNewStream(); + + return absl::OkStatus(); +} + template void GrpcMuxImpl::sendGrpcMessage(RQ& msg_proto, S& sub_state) { if (sub_state.dynamicContextChanged() || !anyRequestSentYetInCurrentStream() || diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h index 37f0c31f1729..46397351407b 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h @@ -105,6 +105,13 @@ class GrpcMuxImpl : public GrpcStreamCallbacks, genericHandleResponse(message->type_url(), *message, control_plane_stats); } + absl::Status + updateMuxSource(Grpc::RawAsyncClientPtr&& primary_async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + CustomConfigValidatorsPtr&& custom_config_validators, Stats::Scope& scope, + BackOffStrategyPtr&& backoff_strategy, + const envoy::config::core::v3::ApiConfigSource& ads_config_source) override; + EdsResourcesCacheOptRef edsResourcesCache() override { return makeOptRefFromPtr(eds_resources_cache_.get()); } @@ -165,12 +172,16 @@ class GrpcMuxImpl : public GrpcStreamCallbacks, } const LocalInfo::LocalInfo& localInfo() const { return local_info_; } + virtual absl::string_view methodName() const PURE; + private: // Helper function to create the grpc_stream_ object. // TODO(adisuissa): this should be removed when envoy.restart_features.xds_failover_support // is deprecated. - std::unique_ptr> - createGrpcStreamObject(GrpcMuxContext& grpc_mux_context); + std::unique_ptr> createGrpcStreamObject( + Grpc::RawAsyncClientPtr&& async_client, Grpc::RawAsyncClientPtr&& failover_async_client, + const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, + BackOffStrategyPtr&& backoff_strategy, const RateLimitSettings& rate_limit_settings); // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a (Delta)DiscoveryRequest). @@ -187,6 +198,7 @@ class GrpcMuxImpl : public GrpcStreamCallbacks, // Invoked when dynamic context parameters change for a resource type. void onDynamicContextUpdate(absl::string_view resource_type_url); + Event::Dispatcher& dispatcher_; // Multiplexes the stream to the primary and failover sources. // TODO(adisuissa): Once envoy.restart_features.xds_failover_support is deprecated, // convert from unique_ptr to GrpcMuxFailover directly. @@ -248,6 +260,11 @@ class GrpcMuxDelta : public GrpcMuxImpl& for_update) override; + +private: + absl::string_view methodName() const override { + return "envoy.service.discovery.v3.AggregatedDiscoveryService.DeltaAggregatedResources"; + } }; class GrpcMuxSotw : public GrpcMuxImpl&) override { ENVOY_BUG(false, "unexpected request for on demand update"); } + +private: + absl::string_view methodName() const override { + return "envoy.service.discovery.v3.AggregatedDiscoveryService.StreamAggregatedResources"; + } }; class NullGrpcMuxImpl : public GrpcMux { @@ -277,6 +299,12 @@ class NullGrpcMuxImpl : public GrpcMux { SubscriptionCallbacks&, OpaqueResourceDecoderSharedPtr, const SubscriptionOptions&) override; + absl::Status updateMuxSource(Grpc::RawAsyncClientPtr&&, Grpc::RawAsyncClientPtr&&, + CustomConfigValidatorsPtr&&, Stats::Scope&, BackOffStrategyPtr&&, + const envoy::config::core::v3::ApiConfigSource&) override { + return absl::UnimplementedError(""); + } + void requestOnDemandUpdate(const std::string&, const absl::flat_hash_set&) override { ENVOY_BUG(false, "unexpected request for on demand update"); } diff --git a/source/extensions/filters/common/expr/context.cc b/source/extensions/filters/common/expr/context.cc index 10438dff4df2..a13a39656af3 100644 --- a/source/extensions/filters/common/expr/context.cc +++ b/source/extensions/filters/common/expr/context.cc @@ -300,13 +300,12 @@ absl::optional PeerWrapper::operator[](CelValue key) const { class FilterStateObjectWrapper : public google::api::expr::runtime::CelMap { public: - FilterStateObjectWrapper(const StreamInfo::FilterState::ObjectReflection* reflection) - : reflection_(reflection) {} + FilterStateObjectWrapper(const StreamInfo::FilterState::Object* object) : object_(object) {} absl::optional operator[](CelValue key) const override { - if (reflection_ == nullptr || !key.IsString()) { + if (object_ == nullptr || !key.IsString()) { return {}; } - auto field_value = reflection_->getField(key.StringOrDie().value()); + auto field_value = object_->getField(key.StringOrDie().value()); return absl::visit(Visitor{}, field_value); } // Default stubs. @@ -325,7 +324,7 @@ class FilterStateObjectWrapper : public google::api::expr::runtime::CelMap { } absl::optional operator()(absl::monostate) { return {}; } }; - const StreamInfo::FilterState::ObjectReflection* reflection_; + const StreamInfo::FilterState::Object* object_; }; absl::optional FilterStateWrapper::operator[](CelValue key) const { @@ -339,17 +338,11 @@ absl::optional FilterStateWrapper::operator[](CelValue key) const { if (cel_state) { return cel_state->exprValue(&arena_, false); } else if (object != nullptr) { - // Attempt to find the reflection object. - auto factory = - Registry::FactoryRegistry::getFactory(value); - if (factory) { - auto reflection = factory->reflect(object); - if (reflection) { - auto* raw_reflection = reflection.release(); - arena_.Own(raw_reflection); - return CelValue::CreateMap( - ProtobufWkt::Arena::Create(&arena_, raw_reflection)); - } + // TODO(wbpcode): the implementation of cannot handle the case where the object has provided + // field support, but callers only want to access the whole object. + if (object->hasFieldSupport()) { + return CelValue::CreateMap( + ProtobufWkt::Arena::Create(&arena_, object)); } absl::optional serialized = object->serializeAsString(); if (serialized.has_value()) { diff --git a/source/extensions/filters/common/ratelimit_config/BUILD b/source/extensions/filters/common/ratelimit_config/BUILD new file mode 100644 index 000000000000..a89b1d3676af --- /dev/null +++ b/source/extensions/filters/common/ratelimit_config/BUILD @@ -0,0 +1,22 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "ratelimit_config_lib", + srcs = ["ratelimit_config.cc"], + hdrs = ["ratelimit_config.h"], + deps = [ + "//envoy/ratelimit:ratelimit_interface", + "//source/common/router:router_ratelimit_lib", + "@com_google_absl//absl/container:inlined_vector", + "@com_google_absl//absl/strings", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/common/ratelimit_config/ratelimit_config.cc b/source/extensions/filters/common/ratelimit_config/ratelimit_config.cc new file mode 100644 index 000000000000..c3fec0f4f691 --- /dev/null +++ b/source/extensions/filters/common/ratelimit_config/ratelimit_config.cc @@ -0,0 +1,143 @@ +#include "source/extensions/filters/common/ratelimit_config/ratelimit_config.h" + +#include "source/common/config/utility.h" +#include "source/common/http/matching/data_impl.h" +#include "source/common/matcher/matcher.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace RateLimit { + +RateLimitPolicy::RateLimitPolicy(const ProtoRateLimit& config, + Server::Configuration::CommonFactoryContext& context, + absl::Status& creation_status, bool no_limit) { + if (config.has_stage() || !config.disable_key().empty()) { + creation_status = + absl::InvalidArgumentError("'stage' field and 'disable_key' field are not supported"); + return; + } + + if (config.has_limit()) { + if (no_limit) { + creation_status = absl::InvalidArgumentError("'limit' field is not supported"); + return; + } + } + + for (const ProtoRateLimit::Action& action : config.actions()) { + switch (action.action_specifier_case()) { + case ProtoRateLimit::Action::ActionSpecifierCase::kSourceCluster: + actions_.emplace_back(new Envoy::Router::SourceClusterAction()); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kDestinationCluster: + actions_.emplace_back(new Envoy::Router::DestinationClusterAction()); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kRequestHeaders: + actions_.emplace_back(new Envoy::Router::RequestHeadersAction(action.request_headers())); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kRemoteAddress: + actions_.emplace_back(new Envoy::Router::RemoteAddressAction()); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kGenericKey: + actions_.emplace_back(new Envoy::Router::GenericKeyAction(action.generic_key())); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kMetadata: + actions_.emplace_back(new Envoy::Router::MetaDataAction(action.metadata())); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kHeaderValueMatch: + actions_.emplace_back( + new Envoy::Router::HeaderValueMatchAction(action.header_value_match(), context)); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kExtension: { + ProtobufMessage::ValidationVisitor& validator = context.messageValidationVisitor(); + auto* factory = + Envoy::Config::Utility::getFactory( + action.extension()); + if (!factory) { + // If no descriptor extension is found, fallback to using HTTP matcher + // input functions. Note that if the same extension name or type was + // dual registered as an extension descriptor and an HTTP matcher input + // function, the descriptor extension takes priority. + Router::RateLimitDescriptorValidationVisitor validation_visitor; + Matcher::MatchInputFactory input_factory(validator, + validation_visitor); + Matcher::DataInputFactoryCb data_input_cb = + input_factory.createDataInput(action.extension()); + actions_.emplace_back(std::make_unique( + action.extension().name(), data_input_cb())); + break; + } + auto message = Envoy::Config::Utility::translateAnyToFactoryConfig( + action.extension().typed_config(), validator, *factory); + Envoy::RateLimit::DescriptorProducerPtr producer = + factory->createDescriptorProducerFromProto(*message, context); + if (producer) { + actions_.emplace_back(std::move(producer)); + } else { + creation_status = absl::InvalidArgumentError( + absl::StrCat("Rate limit descriptor extension failed: ", action.extension().name())); + return; + } + break; + } + case ProtoRateLimit::Action::ActionSpecifierCase::kMaskedRemoteAddress: + actions_.emplace_back(new Router::MaskedRemoteAddressAction(action.masked_remote_address())); + break; + case ProtoRateLimit::Action::ActionSpecifierCase::kQueryParameterValueMatch: + actions_.emplace_back(new Router::QueryParameterValueMatchAction( + action.query_parameter_value_match(), context)); + break; + default: + creation_status = absl::InvalidArgumentError(fmt::format( + "Unsupported rate limit action: {}", static_cast(action.action_specifier_case()))); + return; + } + } +} + +void RateLimitPolicy::populateDescriptors(const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& stream_info, + const std::string& local_service_cluster, + RateLimitDescriptors& descriptors) const { + Envoy::RateLimit::LocalDescriptor descriptor; + for (const Envoy::RateLimit::DescriptorProducerPtr& action : actions_) { + Envoy::RateLimit::DescriptorEntry entry; + if (!action->populateDescriptor(entry, local_service_cluster, headers, stream_info)) { + return; + } + if (!entry.key_.empty()) { + descriptor.entries_.emplace_back(std::move(entry)); + } + } + descriptors.emplace_back(std::move(descriptor)); +} + +RateLimitConfig::RateLimitConfig(const Protobuf::RepeatedPtrField& configs, + Server::Configuration::CommonFactoryContext& context, + absl::Status& creation_status, bool no_limit) { + for (const ProtoRateLimit& config : configs) { + auto descriptor_generator = + std::make_unique(config, context, creation_status, no_limit); + if (!creation_status.ok()) { + return; + } + rate_limit_policies_.emplace_back(std::move(descriptor_generator)); + } +} + +void RateLimitConfig::populateDescriptors(const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& stream_info, + const std::string& local_service_cluster, + RateLimitDescriptors& descriptors) const { + for (const auto& generator : rate_limit_policies_) { + generator->populateDescriptors(headers, stream_info, local_service_cluster, descriptors); + } +} + +} // namespace RateLimit +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/common/ratelimit_config/ratelimit_config.h b/source/extensions/filters/common/ratelimit_config/ratelimit_config.h new file mode 100644 index 000000000000..743e69360c47 --- /dev/null +++ b/source/extensions/filters/common/ratelimit_config/ratelimit_config.h @@ -0,0 +1,57 @@ +#pragma once + +#include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/ratelimit/ratelimit.h" + +#include "source/common/router/router_ratelimit.h" + +#include "absl/container/inlined_vector.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace RateLimit { + +using ProtoRateLimit = envoy::config::route::v3::RateLimit; +using RateLimitDescriptors = std::vector; + +class RateLimitPolicy : Logger::Loggable { +public: + RateLimitPolicy(const ProtoRateLimit& config, + Server::Configuration::CommonFactoryContext& context, + absl::Status& creation_status, bool no_limit = true); + + void populateDescriptors(const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info, + const std::string& local_service_cluster, + RateLimitDescriptors& descriptors) const; + +private: + std::vector actions_; +}; + +class RateLimitConfig : Logger::Loggable { +public: + RateLimitConfig(const Protobuf::RepeatedPtrField& configs, + Server::Configuration::CommonFactoryContext& context, + absl::Status& creation_status, bool no_limit = true); + + bool empty() const { return rate_limit_policies_.empty(); } + + size_t size() const { return rate_limit_policies_.size(); } + + void populateDescriptors(const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info, + const std::string& local_service_cluster, + RateLimitDescriptors& descriptors) const; + +private: + std::vector> rate_limit_policies_; +}; + +} // namespace RateLimit +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index e087cdb1596f..0d35d0542581 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -579,13 +579,11 @@ FilterDataStatus Filter::onData(ProcessorState& state, Buffer::Instance& data, b } else { ENVOY_LOG(trace, "Header processing still in progress -- holding body data"); // We don't know what to do with the body until the response comes back. - // We must buffer it in case we need it when that happens. - // Raise a watermark to prevent a buffer overflow until the response comes back. - // When end_stream is true, we need to StopIterationAndWatermark as well to stop the - // ActiveStream from returning error when the last chunk added to stream buffer exceeds the - // buffer limit. + // We must buffer it in case we need it when that happens. Watermark will be raised when the + // buffered data reaches the buffer's watermark limit. When end_stream is true, we need to + // StopIterationAndWatermark as well to stop the ActiveStream from returning error when the + // last chunk added to stream buffer exceeds the buffer limit. state.setPaused(true); - state.requestWatermark(); return FilterDataStatus::StopIterationAndWatermark; } } @@ -978,9 +976,16 @@ void Filter::sendTrailers(ProcessorState& state, const Http::HeaderMap& trailers auto* trailers_req = state.mutableTrailers(req); MutationUtils::headersToProto(trailers, config_->allowedHeaders(), config_->disallowedHeaders(), *trailers_req->mutable_trailers()); - state.onStartProcessorCall(std::bind(&Filter::onMessageTimeout, this), config_->messageTimeout(), - ProcessorState::CallbackState::TrailersCallback); - ENVOY_LOG(debug, "Sending trailers message"); + + if (observability_mode) { + ENVOY_LOG(debug, "Sending trailers message in observability mode"); + } else { + state.onStartProcessorCall(std::bind(&Filter::onMessageTimeout, this), + config_->messageTimeout(), + ProcessorState::CallbackState::TrailersCallback); + ENVOY_LOG(debug, "Sending trailers message"); + } + sendRequest(std::move(req), false); stats_.stream_msgs_sent_.inc(); } diff --git a/source/extensions/filters/http/ext_proc/processor_state.cc b/source/extensions/filters/http/ext_proc/processor_state.cc index b0c569c7d146..efe972a4fc70 100644 --- a/source/extensions/filters/http/ext_proc/processor_state.cc +++ b/source/extensions/filters/http/ext_proc/processor_state.cc @@ -499,7 +499,7 @@ void EncodingProcessorState::requestWatermark() { void EncodingProcessorState::clearWatermark() { if (watermark_requested_) { - ENVOY_LOG(debug, "Watermark lowered on decoding"); + ENVOY_LOG(debug, "Watermark lowered on encoding"); watermark_requested_ = false; encoder_callbacks_->onEncoderFilterBelowWriteBufferLowWatermark(); } diff --git a/source/extensions/filters/http/local_ratelimit/BUILD b/source/extensions/filters/http/local_ratelimit/BUILD index 88d4042e9809..f20a2b04118e 100644 --- a/source/extensions/filters/http/local_ratelimit/BUILD +++ b/source/extensions/filters/http/local_ratelimit/BUILD @@ -27,6 +27,7 @@ envoy_cc_library( "//source/common/runtime:runtime_lib", "//source/extensions/filters/common/local_ratelimit:local_ratelimit_lib", "//source/extensions/filters/common/ratelimit:ratelimit_lib", + "//source/extensions/filters/common/ratelimit_config:ratelimit_config_lib", "//source/extensions/filters/http/common:pass_through_filter_lib", "//source/extensions/filters/http/common:ratelimit_headers_lib", "@envoy_api//envoy/extensions/common/ratelimit/v3:pkg_cc_proto", diff --git a/source/extensions/filters/http/local_ratelimit/config.cc b/source/extensions/filters/http/local_ratelimit/config.cc index e0990cf4598f..2228619acc75 100644 --- a/source/extensions/filters/http/local_ratelimit/config.cc +++ b/source/extensions/filters/http/local_ratelimit/config.cc @@ -15,12 +15,9 @@ namespace LocalRateLimitFilter { Http::FilterFactoryCb LocalRateLimitFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { - auto& server_context = context.serverFactoryContext(); - FilterConfigSharedPtr filter_config = std::make_shared( - proto_config, server_context.localInfo(), server_context.mainThreadDispatcher(), - server_context.clusterManager(), server_context.singletonManager(), context.scope(), - server_context.runtime()); + FilterConfigSharedPtr filter_config = + std::make_shared(proto_config, context.serverFactoryContext(), context.scope()); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(filter_config)); }; @@ -30,9 +27,7 @@ Router::RouteSpecificFilterConfigConstSharedPtr LocalRateLimitFilterConfig::createRouteSpecificFilterConfigTyped( const envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit& proto_config, Server::Configuration::ServerFactoryContext& context, ProtobufMessage::ValidationVisitor&) { - return std::make_shared( - proto_config, context.localInfo(), context.mainThreadDispatcher(), context.clusterManager(), - context.singletonManager(), context.scope(), context.runtime(), true); + return std::make_shared(proto_config, context, context.scope(), true); } /** diff --git a/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc b/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc index 8356fa49b652..0e12da02bfdd 100644 --- a/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc +++ b/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc @@ -23,10 +23,8 @@ const std::string& PerConnectionRateLimiter::key() { FilterConfig::FilterConfig( const envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit& config, - const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, - Upstream::ClusterManager& cm, Singleton::Manager& singleton_manager, Stats::Scope& scope, - Runtime::Loader& runtime, const bool per_route) - : dispatcher_(dispatcher), status_(toErrorCode(config.status().code())), + Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, const bool per_route) + : dispatcher_(context.mainThreadDispatcher()), status_(toErrorCode(config.status().code())), stats_(generateStats(config.stat_prefix(), scope)), fill_interval_(std::chrono::milliseconds( PROTOBUF_GET_MS_OR_DEFAULT(config.token_bucket(), fill_interval, 0))), @@ -38,7 +36,7 @@ FilterConfig::FilterConfig( config.has_always_consume_default_token_bucket() ? config.always_consume_default_token_bucket().value() : true), - local_info_(local_info), runtime_(runtime), + local_info_(context.localInfo()), runtime_(context.runtime()), filter_enabled_( config.has_filter_enabled() ? absl::optional( @@ -73,20 +71,35 @@ FilterConfig::FilterConfig( throw EnvoyException("local rate limit token bucket must be set for per filter configs"); } + absl::Status creation_status; + rate_limit_config_ = std::make_unique( + config.rate_limits(), context, creation_status); + THROW_IF_NOT_OK_REF(creation_status); + + if (rate_limit_config_->empty()) { + if (!config.descriptors().empty()) { + ENVOY_LOG_FIRST_N( + warn, 20, + "'descriptors' is set but only used by route configuration. Please configure the local " + "rate limit filter using the embedded 'rate_limits' field as route configuration for " + "local rate limits will be ignored in the future."); + } + } + Filters::Common::LocalRateLimit::ShareProviderSharedPtr share_provider; if (config.has_local_cluster_rate_limit()) { if (rate_limit_per_connection_) { throw EnvoyException("local_cluster_rate_limit is set and " "local_rate_limit_per_downstream_connection is set to true"); } - if (!cm.localClusterName().has_value()) { + if (!context.clusterManager().localClusterName().has_value()) { throw EnvoyException("local_cluster_rate_limit is set but no local cluster name is present"); } // If the local cluster name is set then the relevant cluster must exist or the cluster // manager will fail to initialize. share_provider_manager_ = Filters::Common::LocalRateLimit::ShareProviderManager::singleton( - dispatcher, cm, singleton_manager); + dispatcher_, context.clusterManager(), context.singletonManager()); if (!share_provider_manager_) { throw EnvoyException("local_cluster_rate_limit is set but no local cluster is present"); } @@ -95,7 +108,7 @@ FilterConfig::FilterConfig( } rate_limiter_ = std::make_unique( - fill_interval_, max_tokens_, tokens_per_fill_, dispatcher, descriptors_, + fill_interval_, max_tokens_, tokens_per_fill_, dispatcher_, descriptors_, always_consume_default_token_bucket_, std::move(share_provider)); } @@ -137,7 +150,11 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, std::vector descriptors; if (used_config_->hasDescriptors()) { - populateDescriptors(descriptors, headers); + if (used_config_->hasRateLimitConfigs()) { + used_config_->populateDescriptors(headers, decoder_callbacks_->streamInfo(), descriptors); + } else { + populateDescriptors(descriptors, headers); + } } if (ENVOY_LOG_CHECK_LEVEL(debug)) { diff --git a/source/extensions/filters/http/local_ratelimit/local_ratelimit.h b/source/extensions/filters/http/local_ratelimit/local_ratelimit.h index d23f7228f03c..2b3af4ec7fb8 100644 --- a/source/extensions/filters/http/local_ratelimit/local_ratelimit.h +++ b/source/extensions/filters/http/local_ratelimit/local_ratelimit.h @@ -19,6 +19,7 @@ #include "source/common/runtime/runtime_protos.h" #include "source/extensions/filters/common/local_ratelimit/local_ratelimit_impl.h" #include "source/extensions/filters/common/ratelimit/ratelimit.h" +#include "source/extensions/filters/common/ratelimit_config/ratelimit_config.h" #include "source/extensions/filters/http/common/pass_through_filter.h" namespace Envoy { @@ -69,12 +70,12 @@ class PerConnectionRateLimiter : public StreamInfo::FilterState::Object { /** * Global configuration for the HTTP local rate limit filter. */ -class FilterConfig : public Router::RouteSpecificFilterConfig { +class FilterConfig : public Router::RouteSpecificFilterConfig, + Logger::Loggable { public: FilterConfig(const envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit& config, - const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, - Upstream::ClusterManager& cm, Singleton::Manager& singleton_manager, - Stats::Scope& scope, Runtime::Loader& runtime, bool per_route = false); + Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, + const bool per_route = false); ~FilterConfig() override { // Ensure that the LocalRateLimiterImpl instance will be destroyed on the thread where its inner // timer is created and running. @@ -113,6 +114,18 @@ class FilterConfig : public Router::RouteSpecificFilterConfig { return rate_limited_grpc_status_; } + bool hasRateLimitConfigs() const { + ASSERT(rate_limit_config_ != nullptr); + return !rate_limit_config_->empty(); + } + + void populateDescriptors(const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info, + Filters::Common::RateLimit::RateLimitDescriptors& descriptors) const { + ASSERT(rate_limit_config_ != nullptr); + rate_limit_config_->populateDescriptors(headers, info, local_info_.clusterName(), descriptors); + } + private: friend class FilterTest; @@ -150,6 +163,7 @@ class FilterConfig : public Router::RouteSpecificFilterConfig { const bool enable_x_rate_limit_headers_; const envoy::extensions::common::ratelimit::v3::VhRateLimitsOptions vh_rate_limits_; const absl::optional rate_limited_grpc_status_; + std::unique_ptr rate_limit_config_; }; using FilterConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/http/oauth2/filter.cc b/source/extensions/filters/http/oauth2/filter.cc index 12657167f4bf..4c8bd1da6b62 100644 --- a/source/extensions/filters/http/oauth2/filter.cc +++ b/source/extensions/filters/http/oauth2/filter.cc @@ -422,9 +422,10 @@ Http::FilterHeadersStatus OAuth2Filter::decodeHeaders(Http::RequestHeaderMap& he original_request_url_ = result.original_request_url_; auth_code_ = result.auth_code_; - Formatter::FormatterImpl formatter(config_->redirectUri()); + Formatter::FormatterPtr formatter = THROW_OR_RETURN_VALUE( + Formatter::FormatterImpl::create(config_->redirectUri()), Formatter::FormatterPtr); const auto redirect_uri = - formatter.formatWithContext({&headers}, decoder_callbacks_->streamInfo()); + formatter->formatWithContext({&headers}, decoder_callbacks_->streamInfo()); oauth_client_->asyncGetAccessToken(auth_code_, config_->clientId(), config_->clientSecret(), redirect_uri, config_->authType()); @@ -519,9 +520,10 @@ void OAuth2Filter::redirectToOAuthServer(Http::RequestHeaderMap& headers) const absl::StrCat(stateParamsUrl, "=", escaped_url, "&", stateParamsNonce, "=", nonce); const std::string escaped_state = Http::Utility::PercentEncoding::urlEncodeQueryParameter(state); - Formatter::FormatterImpl formatter(config_->redirectUri()); + Formatter::FormatterPtr formatter = THROW_OR_RETURN_VALUE( + Formatter::FormatterImpl::create(config_->redirectUri()), Formatter::FormatterPtr); const auto redirect_uri = - formatter.formatWithContext({&headers}, decoder_callbacks_->streamInfo()); + formatter->formatWithContext({&headers}, decoder_callbacks_->streamInfo()); const std::string escaped_redirect_uri = Http::Utility::PercentEncoding::urlEncodeQueryParameter(redirect_uri); diff --git a/source/extensions/filters/http/rate_limit_quota/client_impl.cc b/source/extensions/filters/http/rate_limit_quota/client_impl.cc index 876060592eb3..e8f9c49a11e5 100644 --- a/source/extensions/filters/http/rate_limit_quota/client_impl.cc +++ b/source/extensions/filters/http/rate_limit_quota/client_impl.cc @@ -100,8 +100,18 @@ void RateLimitClientImpl::onReceiveMessage(RateLimitQuotaResponsePtr&& response) switch (action.bucket_action_case()) { case envoy::service::rate_limit_quota::v3::RateLimitQuotaResponse_BucketAction:: kQuotaAssignmentAction: { - quota_buckets_[bucket_id]->cached_action = action; + absl::optional cached_action = quota_buckets_[bucket_id]->cached_action; quota_buckets_[bucket_id]->current_assignment_time = time_source_.monotonicTime(); + + if (cached_action.has_value() && + Protobuf::util::MessageDifferencer::Equals(*cached_action, action)) { + ENVOY_LOG(debug, + "Cached action matches the incoming response so only TTL is updated for bucket " + "id: {}", + bucket_id); + break; + } + quota_buckets_[bucket_id]->cached_action = action; if (quota_buckets_[bucket_id]->cached_action->has_quota_assignment_action()) { auto rate_limit_strategy = quota_buckets_[bucket_id] ->cached_action->quota_assignment_action() diff --git a/source/extensions/filters/network/dubbo_proxy/hessian_utils.h b/source/extensions/filters/network/dubbo_proxy/hessian_utils.h index da30248da3b2..8385eb37c0a4 100644 --- a/source/extensions/filters/network/dubbo_proxy/hessian_utils.h +++ b/source/extensions/filters/network/dubbo_proxy/hessian_utils.h @@ -5,16 +5,7 @@ #include "envoy/buffer/buffer.h" #include "absl/strings/string_view.h" - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif #include "hessian2/basic_codec/object_codec.hpp" -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - #include "hessian2/codec.hpp" #include "hessian2/object.hpp" #include "hessian2/reader.hpp" diff --git a/source/extensions/filters/network/ratelimit/ratelimit.cc b/source/extensions/filters/network/ratelimit/ratelimit.cc index ba9a1aa3a523..426d712a728b 100644 --- a/source/extensions/filters/network/ratelimit/ratelimit.cc +++ b/source/extensions/filters/network/ratelimit/ratelimit.cc @@ -30,7 +30,8 @@ Config::Config(const envoy::extensions::filters::network::ratelimit::v3::RateLim for (const auto& entry : descriptor.entries()) { new_descriptor.entries_.push_back({entry.key(), entry.value()}); substitution_formatters_.push_back( - std::make_unique(entry.value(), false)); + THROW_OR_RETURN_VALUE(Formatter::FormatterImpl::create(entry.value(), false), + std::unique_ptr)); } original_descriptors_.push_back(new_descriptor); } diff --git a/source/extensions/filters/network/redis_proxy/router_impl.cc b/source/extensions/filters/network/redis_proxy/router_impl.cc index 09621e04facd..72c96b345663 100644 --- a/source/extensions/filters/network/redis_proxy/router_impl.cc +++ b/source/extensions/filters/network/redis_proxy/router_impl.cc @@ -136,7 +136,7 @@ void PrefixRoutes::formatKey(std::string& key, std::string redis_key_formatter, redis_key_formatter = absl::StrReplaceAll( redis_key_formatter, {{redis_key_formatter_command_, redis_key_to_be_replaced_}}); } - auto providers = Formatter::SubstitutionFormatParser::parse(redis_key_formatter); + auto providers = *Formatter::SubstitutionFormatParser::parse(redis_key_formatter); std::string formatted_key; for (Formatter::FormatterProviderPtr& provider : providers) { auto provider_formatted_key = provider->formatValueWithContext({}, stream_info); diff --git a/source/extensions/health_checkers/grpc/health_checker_impl.cc b/source/extensions/health_checkers/grpc/health_checker_impl.cc index f9df90eaa164..e5873e8ec041 100644 --- a/source/extensions/health_checkers/grpc/health_checker_impl.cc +++ b/source/extensions/health_checkers/grpc/health_checker_impl.cc @@ -196,15 +196,8 @@ void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::onInterval() { request_encoder_ = &client_->newStream(*this); request_encoder_->getStream().addCallbacks(*this); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif const std::string& authority = getHostname(host_, parent_.authority_value_, parent_.cluster_.info()); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif auto headers_message = Grpc::Common::prepareHeaders(authority, parent_.service_method_.service()->full_name(), parent_.service_method_.name(), absl::nullopt); diff --git a/source/extensions/health_checkers/http/health_checker_impl.h b/source/extensions/health_checkers/http/health_checker_impl.h index d4a03688c13d..939062b7365f 100644 --- a/source/extensions/health_checkers/http/health_checker_impl.h +++ b/source/extensions/health_checkers/http/health_checker_impl.h @@ -141,14 +141,7 @@ class HttpHealthCheckerImpl : public HealthCheckerImplBase { Http::CodecClientPtr client_; Http::ResponseHeaderMapPtr response_headers_; Buffer::InstancePtr response_body_; -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif const std::string& hostname_; -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif Network::ConnectionInfoProviderSharedPtr local_connection_info_provider_; // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. const Http::Protocol protocol_; diff --git a/source/extensions/health_checkers/thrift/thrift.h b/source/extensions/health_checkers/thrift/thrift.h index 329a01e9d08d..9c75e9df56d7 100644 --- a/source/extensions/health_checkers/thrift/thrift.h +++ b/source/extensions/health_checkers/thrift/thrift.h @@ -57,14 +57,7 @@ class ThriftHealthChecker : public Upstream::HealthCheckerImplBase { private: ThriftHealthChecker& parent_; -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdangling-reference" -#endif const std::string& hostname_; -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif ClientPtr client_; bool expect_close_{}; }; diff --git a/source/extensions/network/dns_resolver/apple/apple_dns_impl.h b/source/extensions/network/dns_resolver/apple/apple_dns_impl.h index d8f01bd7360f..7b32cb468017 100644 --- a/source/extensions/network/dns_resolver/apple/apple_dns_impl.h +++ b/source/extensions/network/dns_resolver/apple/apple_dns_impl.h @@ -100,6 +100,8 @@ class AppleDnsResolverImpl : public DnsResolver, protected Logger::Loggable(dns_name, dns_lookup_family, callback, mutex_); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.getaddrinfo_num_retries") && - config_.has_num_retries()) { - // + 1 to include the initial query. - pending_queries_.push_back({std::move(new_query), config_.num_retries().value() + 1}); - } else { - pending_queries_.push_back({std::move(new_query), absl::nullopt}); + auto new_query = std::make_unique(dns_name, dns_lookup_family, callback); + ActiveDnsQuery* active_query; + { + absl::MutexLock guard(&mutex_); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.getaddrinfo_num_retries") && + config_.has_num_retries()) { + // + 1 to include the initial query. + pending_queries_.push_back({std::move(new_query), config_.num_retries().value() + 1}); + } else { + pending_queries_.push_back({std::move(new_query), absl::nullopt}); + } + active_query = pending_queries_.back().pending_query_.get(); } - return pending_queries_.back().pending_query_.get(); + active_query->addTrace(static_cast(GetAddrInfoTrace::NotStarted)); + return active_query; } std::pair> @@ -117,7 +122,7 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { ENVOY_LOG(debug, "starting getaddrinfo resolver thread"); while (true) { - PendingQuerySharedPtr next_query; + std::unique_ptr next_query; absl::optional num_retries; const bool reresolve = Runtime::runtimeFeatureEnabled("envoy.reloadable_features.dns_reresolve_on_eai_again"); @@ -134,10 +139,10 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { } PendingQueryInfo pending_query_info = std::move(pending_queries_.front()); - next_query = pending_query_info.pending_query_; + next_query = std::move(pending_query_info.pending_query_); num_retries = pending_query_info.num_retries_; pending_queries_.pop_front(); - if (reresolve && next_query->cancelled_) { + if (reresolve && next_query->isCancelled()) { continue; } } @@ -148,6 +153,7 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { std::pair> response; std::string details; { + next_query->addTrace(static_cast(GetAddrInfoTrace::Starting)); addrinfo hints; memset(&hints, 0, sizeof(hints)); hints.ai_flags = AI_ADDRCONFIG; @@ -161,25 +167,34 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { &hints, &addrinfo_result_do_not_use); auto addrinfo_wrapper = AddrInfoWrapper(addrinfo_result_do_not_use); if (rc.return_value_ == 0) { + next_query->addTrace(static_cast(GetAddrInfoTrace::Success)); response = processResponse(*next_query, addrinfo_wrapper.get()); } else if (reresolve && rc.return_value_ == EAI_AGAIN) { - absl::MutexLock guard(&mutex_); if (num_retries.has_value()) { (*num_retries)--; } if (!num_retries.has_value()) { ENVOY_LOG(debug, "retrying query [{}]", next_query->dns_name_); - pending_queries_.push_back({std::move(next_query), absl::nullopt}); + next_query->addTrace(static_cast(GetAddrInfoTrace::Retrying)); + { + absl::MutexLock guard(&mutex_); + pending_queries_.push_back({std::move(next_query), absl::nullopt}); + } continue; } if (*num_retries > 0) { ENVOY_LOG(debug, "retrying query [{}], num_retries: {}", next_query->dns_name_, *num_retries); - pending_queries_.push_back({std::move(next_query), *num_retries}); + next_query->addTrace(static_cast(GetAddrInfoTrace::Retrying)); + { + absl::MutexLock guard(&mutex_); + pending_queries_.push_back({std::move(next_query), *num_retries}); + } continue; } ENVOY_LOG(debug, "not retrying query [{}] because num_retries: {}", next_query->dns_name_, *num_retries); + next_query->addTrace(static_cast(GetAddrInfoTrace::DoneRetrying)); response = std::make_pair(ResolutionStatus::Failure, std::list()); } else if (treat_nodata_noname_as_success && (rc.return_value_ == EAI_NONAME || rc.return_value_ == EAI_NODATA)) { @@ -190,10 +205,12 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { // https://github.com/envoyproxy/envoy/blob/099d85925b32ce8bf06e241ee433375a0a3d751b/source/extensions/network/dns_resolver/cares/dns_impl.h#L109-L111. ENVOY_LOG(debug, "getaddrinfo for host={} has no results rc={}", next_query->dns_name_, gai_strerror(rc.return_value_)); + next_query->addTrace(static_cast(GetAddrInfoTrace::NoResult)); response = std::make_pair(ResolutionStatus::Completed, std::list()); } else { ENVOY_LOG(debug, "getaddrinfo failed for host={} with rc={} errno={}", next_query->dns_name_, gai_strerror(rc.return_value_), errorDetails(rc.errno_)); + next_query->addTrace(static_cast(GetAddrInfoTrace::Failed)); response = std::make_pair(ResolutionStatus::Failure, std::list()); } details = gai_strerror(rc.return_value_); @@ -201,9 +218,11 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { dispatcher_.post([finished_query = std::move(next_query), response = std::move(response), details = std::string(details)]() mutable { - if (finished_query->cancelled_) { + if (finished_query->isCancelled()) { + finished_query->addTrace(static_cast(GetAddrInfoTrace::Cancelled)); ENVOY_LOG(debug, "dropping cancelled query [{}]", finished_query->dns_name_); } else { + finished_query->addTrace(static_cast(GetAddrInfoTrace::Callback)); finished_query->callback_(response.first, std::move(details), std::move(response.second)); } }); diff --git a/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h b/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h index 56f71a945c87..2357fdb4ace6 100644 --- a/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h +++ b/source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h @@ -12,6 +12,19 @@ namespace Network { DECLARE_FACTORY(GetAddrInfoDnsResolverFactory); +// Trace information for getaddrinfo. +enum class GetAddrInfoTrace : uint8_t { + NotStarted = 0, + Starting = 1, + Success = 2, + Failed = 3, + NoResult = 4, + Retrying = 5, + DoneRetrying = 6, + Cancelled = 7, + Callback = 8, +}; + // This resolver uses getaddrinfo() on a dedicated resolution thread. Thus, it is only suitable // currently for relatively low rate resolutions. In the future a thread pool could be added if // desired. @@ -34,28 +47,48 @@ class GetAddrInfoDnsResolver : public DnsResolver, public Logger::Loggable string_traces; + string_traces.reserve(traces_.size()); + std::transform(traces_.begin(), traces_.end(), std::back_inserter(string_traces), + [](const Trace& trace) { + return absl::StrCat(trace.trace_, "=", + trace.time_.time_since_epoch().count()); + }); + return absl::StrJoin(string_traces, ","); + } + + bool isCancelled() { + absl::MutexLock lock(&mutex_); + return cancelled_; + } + + absl::Mutex mutex_; const std::string dns_name_; const DnsLookupFamily dns_lookup_family_; ResolveCb callback_; bool cancelled_{false}; + std::vector traces_; }; - // Must be a shared_ptr for passing around via post. - using PendingQuerySharedPtr = std::shared_ptr; struct PendingQueryInfo { - PendingQuerySharedPtr pending_query_; + std::unique_ptr pending_query_; // Empty means it will retry indefinitely until it succeeds. absl::optional num_retries_; }; diff --git a/source/server/server.cc b/source/server/server.cc index 826c6dc4910c..d29713c65aad 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -720,6 +720,8 @@ absl::Status InstanceBase::initializeOrThrow(Network::Address::InstanceConstShar auto typed_admin = dynamic_cast(admin_.get()); RELEASE_ASSERT(typed_admin != nullptr, "Admin implementation is not an AdminImpl."); initial_config.initAdminAccessLog(bootstrap_, typed_admin->factoryContext()); + ENVOY_LOG(info, "Starting admin HTTP server at {}", + initial_config.admin().address()->asString()); admin_->startHttpListener(initial_config.admin().accessLogs(), initial_config.admin().address(), initial_config.admin().socketOptions()); #else diff --git a/test/common/common/BUILD b/test/common/common/BUILD index d15e11f00fc6..f1d14259d041 100644 --- a/test/common/common/BUILD +++ b/test/common/common/BUILD @@ -603,6 +603,8 @@ envoy_cc_test( deps = [ "//envoy/common:execution_context", "//source/common/api:api_lib", + "//source/common/http:conn_manager_lib", + "//source/common/http:filter_manager_lib", "//test/mocks:common_lib", "//test/mocks/stream_info:stream_info_mocks", ], diff --git a/test/common/common/execution_context_test.cc b/test/common/common/execution_context_test.cc index 8aed2f00eaf8..66dd126ce4aa 100644 --- a/test/common/common/execution_context_test.cc +++ b/test/common/common/execution_context_test.cc @@ -1,9 +1,13 @@ #include #include "envoy/common/execution_context.h" +#include "envoy/http/filter_factory.h" #include "source/common/api/api_impl.h" #include "source/common/common/scope_tracker.h" +#include "source/common/http/conn_manager_impl.h" +#include "source/common/http/filter_manager.h" +#include "source/common/tracing/null_span_impl.h" #include "test/mocks/common.h" #include "test/mocks/stream_info/mocks.h" @@ -14,8 +18,21 @@ namespace Envoy { +thread_local const Http::FilterContext* current_filter_context = nullptr; + class TestExecutionContext : public ExecutionContext { public: + Envoy::Cleanup onScopeEnter(Envoy::Tracing::Span*) override { return Envoy::Cleanup::Noop(); } + + Envoy::Cleanup onScopeEnter(const Http::FilterContext* filter_context) override { + if (filter_context == nullptr) { + return Envoy::Cleanup::Noop(); + } + const Http::FilterContext* old_filter_context = current_filter_context; + current_filter_context = filter_context; + return Envoy::Cleanup([old_filter_context]() { current_filter_context = old_filter_context; }); + } + int activationDepth() const { return activation_depth_; } int activationGenerations() const { return activation_generations_; } @@ -39,6 +56,11 @@ class TestExecutionContext : public ExecutionContext { class ExecutionContextTest : public testing::Test { public: + static void SetUpTestCase() { + EXPECT_FALSE(ExecutionContext::isEnabled()); + ExecutionContext::setEnabled(true); + } + ExecutionContextTest() { ON_CALL(tracked_object_, trackedStream()) .WillByDefault(testing::Return(OptRef(stream_info_))); @@ -114,7 +136,6 @@ TEST_F(ExecutionContextTest, DisjointScopes) { } TEST_F(ExecutionContextTest, InScopeTrackerScopeState) { - Api::ApiPtr api(Api::createApiForTest()); Event::DispatcherPtr dispatcher(api->allocateDispatcher("test_thread")); EXPECT_CALL(tracked_object_, trackedStream()) @@ -137,4 +158,54 @@ TEST_F(ExecutionContextTest, InScopeTrackerScopeState) { { ScopeTrackerScopeState scope(&tracked_object_, *dispatcher); } } +TEST_F(ExecutionContextTest, NoopScope) { + OptRef null_stream_info; + Http::FilterContext* null_filter_context = nullptr; + Tracing::Span* null_tracing_span = nullptr; + ENVOY_EXECUTION_SCOPE(null_stream_info, null_filter_context); + ENVOY_EXECUTION_SCOPE(null_stream_info, null_tracing_span); + + Http::FilterContext filter_context; + ENVOY_EXECUTION_SCOPE(null_stream_info, &filter_context); + ENVOY_EXECUTION_SCOPE(null_stream_info, &Tracing::NullSpan::instance()); + + setWithContext(); + EXPECT_CALL(tracked_object_, trackedStream()) + .Times(2) + .WillRepeatedly(testing::Return(OptRef(stream_info_))); + ENVOY_EXECUTION_SCOPE(tracked_object_.trackedStream(), null_filter_context); + ENVOY_EXECUTION_SCOPE(tracked_object_.trackedStream(), null_tracing_span); +} + +TEST_F(ExecutionContextTest, FilterScope) { + setWithContext(); + EXPECT_CALL(tracked_object_, trackedStream()) + .Times(2) + .WillRepeatedly(testing::Return(OptRef(stream_info_))); + + Http::FilterContext outer_filter_context{"outer_filter"}; + ENVOY_EXECUTION_SCOPE(tracked_object_.trackedStream(), &outer_filter_context); + EXPECT_EQ(current_filter_context, &outer_filter_context); + + { + Http::FilterContext inner_filter_context{"inner_filter"}; + ENVOY_EXECUTION_SCOPE(tracked_object_.trackedStream(), &inner_filter_context); + EXPECT_EQ(current_filter_context, &inner_filter_context); + } + + EXPECT_EQ(current_filter_context, &outer_filter_context); +} + +// Make sure source/common/http/conn_manager_impl.cc compiles with ENVOY_ENABLE_EXECUTION_CONTEXT. +TEST_F(ExecutionContextTest, ConnectionManagerImplCompiles) { + const Http::ConnectionManagerImpl* impl = nullptr; + EXPECT_EQ(impl, impl); +} + +// Make sure source/common/http/filter_manager.cc compiles with ENVOY_ENABLE_EXECUTION_CONTEXT. +TEST_F(ExecutionContextTest, FilterManagerCompiles) { + const Http::FilterManager* manager = nullptr; + EXPECT_EQ(manager, manager); +} + } // namespace Envoy diff --git a/test/common/config/BUILD b/test/common/config/BUILD index bc68571f940a..9111a9019602 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -144,6 +144,7 @@ envoy_cc_test( "//test/mocks/upstream:thread_local_cluster_mocks", "//test/test_common:environment_lib", "//test/test_common:logging_lib", + "//test/test_common:status_utility_lib", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@com_github_cncf_xds//udpa/type/v1:pkg_cc_proto", diff --git a/test/common/config/utility_test.cc b/test/common/config/utility_test.cc index b3945df1e62d..98735c89c211 100644 --- a/test/common/config/utility_test.cc +++ b/test/common/config/utility_test.cc @@ -20,6 +20,7 @@ #include "test/mocks/upstream/thread_local_cluster.h" #include "test/test_common/environment.h" #include "test/test_common/logging.h" +#include "test/test_common/status_utility.h" #include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" @@ -29,6 +30,7 @@ #include "udpa/type/v1/typed_struct.pb.h" #include "xds/type/v3/typed_struct.pb.h" +using Envoy::StatusHelpers::HasStatusMessage; using testing::ContainsRegex; using testing::Eq; using testing::HasSubstr; @@ -894,6 +896,43 @@ TEST(UtilityTest, GetGrpcControlPlane) { } } +TEST(UtilityTest, ValidateTerminalFiltersSucceeds) { + EXPECT_OK(Utility::validateTerminalFilters("filter_name", "filter_type", "chain_type", + /*is_terminal_filter=*/true, + /*last_filter_in_current_config=*/true)); + EXPECT_OK(Utility::validateTerminalFilters("filter_name", "filter_type", "chain_type", + /*is_terminal_filter=*/false, + /*last_filter_in_current_config=*/false)); +} + +TEST(UtilityTest, ValidateTerminalFilterFailsWithMisplacedTerminalFilter) { + EXPECT_THAT( + Utility::validateTerminalFilters("filter_name", "filter_type", "chain_type", + /*is_terminal_filter=*/true, + /*last_filter_in_current_config=*/false), + HasStatusMessage("Error: terminal filter named filter_name of type filter_type must be the " + "last filter in a chain_type filter chain.")); +} + +TEST(UtilityTest, ValidateTerminalFilterFailsWithMissingTerminalFilter) { + EXPECT_THAT(Utility::validateTerminalFilters("filter_name", "filter_type", "chain_type", + /*is_terminal_filter=*/false, + /*last_filter_in_current_config=*/true), + HasStatusMessage("Error: non-terminal filter named filter_name of type " + "filter_type is the last filter in a chain_type filter chain.")); +} + +TEST(UtilityTest, ValidateTerminalFilterFailsWithMissingUpstreamTerminalFilter) { + EXPECT_THAT( + Utility::validateTerminalFilters("filter_name", "filter_type", "router upstream http", + /*is_terminal_filter=*/false, + /*last_filter_in_current_config=*/true), + HasStatusMessage("Error: non-terminal filter named filter_name of type " + "filter_type is the last filter in a router upstream http filter chain. " + "When upstream_http_filters are specified, they must explicitly end with an " + "UpstreamCodec filter.")); +} + } // namespace } // namespace Config } // namespace Envoy diff --git a/test/common/formatter/substitution_formatter_fuzz_test.cc b/test/common/formatter/substitution_formatter_fuzz_test.cc index 7540c37f9387..79f676d8108f 100644 --- a/test/common/formatter/substitution_formatter_fuzz_test.cc +++ b/test/common/formatter/substitution_formatter_fuzz_test.cc @@ -39,7 +39,13 @@ DEFINE_PROTO_FUZZER(const test::common::substitution::TestCase& input) { { Formatter::FormatterPtr formatter; try { - formatter = std::make_unique(input.format()); + auto formatter_or_error = Formatter::FormatterImpl::create(input.format(), false); + if (!formatter_or_error.status().ok()) { + ENVOY_LOG_MISC(debug, "TEXT formatter failed, EnvoyException: {}", + formatter_or_error.status().message()); + return; + } + formatter = std::move(*formatter_or_error); } catch (const EnvoyException& e) { ENVOY_LOG_MISC(debug, "TEXT formatter failed, EnvoyException: {}", e.what()); return; diff --git a/test/common/formatter/substitution_formatter_speed_test.cc b/test/common/formatter/substitution_formatter_speed_test.cc index 9e39bee30674..0c006993484e 100644 --- a/test/common/formatter/substitution_formatter_speed_test.cc +++ b/test/common/formatter/substitution_formatter_speed_test.cc @@ -11,25 +11,6 @@ namespace Envoy { namespace { -std::unique_ptr makeLegacyJsonFormatter(bool typed) { - ProtobufWkt::Struct JsonLogFormat; - const std::string format_yaml = R"EOF( - remote_address: '%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%' - start_time: '%START_TIME(%Y/%m/%dT%H:%M:%S%z %s)%' - method: '%REQ(:METHOD)%' - url: '%REQ(X-FORWARDED-PROTO)%://%REQ(:AUTHORITY)%%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%' - protocol: '%PROTOCOL%' - response_code: '%RESPONSE_CODE%' - bytes_sent: '%BYTES_SENT%' - duration: '%DURATION%' - referer: '%REQ(REFERER)%' - user-agent: '%REQ(USER-AGENT)%' - )EOF"; - TestUtility::loadFromYaml(format_yaml, JsonLogFormat); - return std::make_unique(JsonLogFormat, typed, false, - false); -} - std::unique_ptr makeJsonFormatter() { ProtobufWkt::Struct JsonLogFormat; const std::string format_yaml = R"EOF( @@ -87,7 +68,7 @@ static void BM_AccessLogFormatterSetup(benchmark::State& state) { for (auto _ : state) { // NOLINT: Silences warning about dead store std::unique_ptr formatter = - std::make_unique(LogFormat, false); + *Envoy::Formatter::FormatterImpl::create(LogFormat, false); } } BENCHMARK(BM_AccessLogFormatterSetup); @@ -104,7 +85,7 @@ static void BM_AccessLogFormatter(benchmark::State& state) { "s%RESPONSE_CODE% %BYTES_SENT% %DURATION% %REQ(REFERER)% \"%REQ(USER-AGENT)%\" - - -\n"; std::unique_ptr formatter = - std::make_unique(LogFormat, false); + *Envoy::Formatter::FormatterImpl::create(LogFormat, false); size_t output_bytes = 0; for (auto _ : state) { // NOLINT: Silences warning about dead store @@ -145,34 +126,6 @@ static void BM_TypedStructAccessLogFormatter(benchmark::State& state) { } BENCHMARK(BM_TypedStructAccessLogFormatter); -// NOLINTNEXTLINE(readability-identifier-naming) -static void BM_LegacyJsonAccessLogFormatter(benchmark::State& state) { - testing::NiceMock time_system; - std::unique_ptr stream_info = makeStreamInfo(time_system); - auto json_formatter = makeLegacyJsonFormatter(false); - - size_t output_bytes = 0; - for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += json_formatter->formatWithContext({}, *stream_info).length(); - } - benchmark::DoNotOptimize(output_bytes); -} -BENCHMARK(BM_LegacyJsonAccessLogFormatter); - -// NOLINTNEXTLINE(readability-identifier-naming) -static void BM_LegacyTypedJsonAccessLogFormatter(benchmark::State& state) { - testing::NiceMock time_system; - std::unique_ptr stream_info = makeStreamInfo(time_system); - auto json_formatter = makeLegacyJsonFormatter(true); - - size_t output_bytes = 0; - for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += json_formatter->formatWithContext({}, *stream_info).length(); - } - benchmark::DoNotOptimize(output_bytes); -} -BENCHMARK(BM_LegacyTypedJsonAccessLogFormatter); - // NOLINTNEXTLINE(readability-identifier-naming) static void BM_JsonAccessLogFormatter(benchmark::State& state) { testing::NiceMock time_system; diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 41b3e2fcbac4..9c058bdf46ef 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -123,18 +123,10 @@ class TestSerializedStringFilterState : public StreamInfo::FilterState::Object { message->set_value(raw_string_ + " By TYPED"); return message; } - -private: - std::string raw_string_; - friend class TestSerializedStringReflection; -}; - -class TestSerializedStringReflection : public StreamInfo::FilterState::ObjectReflection { -public: - TestSerializedStringReflection(const TestSerializedStringFilterState* data) : data_(data) {} + bool hasFieldSupport() const override { return true; } FieldType getField(absl::string_view field_name) const override { if (field_name == "test_field") { - return data_->raw_string_; + return raw_string_; } else if (field_name == "test_num") { return 137; } @@ -142,25 +134,9 @@ class TestSerializedStringReflection : public StreamInfo::FilterState::ObjectRef } private: - const TestSerializedStringFilterState* data_; -}; - -class TestSerializedStringFilterStateFactory : public StreamInfo::FilterState::ObjectFactory { -public: - std::string name() const override { return "test_key"; } - std::unique_ptr - createFromBytes(absl::string_view) const override { - return nullptr; - } - std::unique_ptr - reflect(const StreamInfo::FilterState::Object* data) const override { - return std::make_unique( - dynamic_cast(data)); - } + std::string raw_string_; }; -REGISTER_FACTORY(TestSerializedStringFilterStateFactory, StreamInfo::FilterState::ObjectFactory); - // Test tests multiple versions of variadic template method parseSubcommand // extracting tokens. TEST(SubstitutionFormatParser, commandParser) { @@ -4616,21 +4592,21 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const std::string format = "{{%PROTOCOL%}} %RESP(not_exist)%++%RESP(test)% " "%REQ(FIRST?SECOND)% %RESP(FIRST?SECOND)%" "\t@%TRAILER(THIRD)%@\t%TRAILER(TEST?TEST-2)%[]"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); EXPECT_EQ("{{HTTP/1.1}} -++test GET PUT\t@POST@\ttest-2[]", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { NiceMock stream_info; const std::string format = "{}*JUST PLAIN string]"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); - EXPECT_EQ(format, formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ(format, formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4638,9 +4614,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const std::string format = "%REQ(first):3%|%REQ(first):1%|%RESP(first?second):2%|%REQ(first):" "10%|%TRAILER(second?third):3%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); - EXPECT_EQ("GET|G|PU|GET|POS", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("GET|G|PU|GET|POS", formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4651,10 +4627,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { EXPECT_CALL(Const(stream_info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); const std::string format = "%DYNAMIC_METADATA(com.test:test_key)%|%DYNAMIC_METADATA(com.test:" "test_obj)%|%DYNAMIC_METADATA(com.test:test_obj:inner_key)%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("test_value|{\"inner_key\":\"inner_value\"}|inner_value", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4670,10 +4646,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { StreamInfo::FilterState::LifeSpan::FilterChain); const std::string format = "%FILTER_STATE(testing)%|%FILTER_STATE(serialized)%|" "%FILTER_STATE(testing):8%|%FILTER_STATE(nonexisting)%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("\"test_value\"|-|\"test_va|-", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4685,11 +4661,11 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { time_t expected_time_in_epoch = 1522280158; SystemTime time = std::chrono::system_clock::from_time_t(expected_time_in_epoch); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(time)); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ(fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", expected_time_in_epoch), - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4705,11 +4681,11 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { SystemTime time = std::chrono::system_clock::from_time_t(expected_time_in_epoch); EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(time)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ(fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", expected_time_in_epoch), - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4725,11 +4701,11 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { SystemTime time = std::chrono::system_clock::from_time_t(expected_time_in_epoch); EXPECT_CALL(*connection_info, expirationPeerCertificate()).WillRepeatedly(Return(time)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ(fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", expected_time_in_epoch), - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4741,10 +4717,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const time_t test_epoch = 0; const SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(time)); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("1970/01/01|0|bad_format|1970-01-01T00:00:00.000Z|000000000.0.00.000", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4754,9 +4730,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { "%START_TIME(%s.%3f)%|%START_TIME(%s.%4f)%|%START_TIME(%s.%5f)%|%START_TIME(%s.%6f)%"; const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(start_time)); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("1522796769.123|1522796769.1234|1522796769.12345|1522796769.123456", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4765,10 +4741,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { "%START_TIME(segment1:%s.%3f|segment2:%s.%4f|seg3:%s.%6f|%s-%3f-asdf-%9f|.%7f:segm5:%Y)%"; const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(start_time)); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("segment1:1522796769.123|segment2:1522796769.1234|seg3:1522796769.123456|1522796769-" "123-asdf-123456000|.1234560:segm5:2018", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4778,9 +4754,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const std::string format = "%START_TIME(%%%%|%%%%%f|%s%%%%%3f|%1f%%%%%s)%"; const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillOnce(Return(start_time)); - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_EQ("%%|%%123456000|1522796769%%123|1%%1522796769", - formatter.formatWithContext(formatter_context, stream_info)); + formatter->formatWithContext(formatter_context, stream_info)); } // The %E formatting option in Absl::FormatTime() behaves differently for non Linux platforms. @@ -4794,8 +4770,8 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const std::string format = "%START_TIME(%E4n)%"; const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillOnce(Return(start_time)); - FormatterImpl formatter(format, false); - EXPECT_EQ("%E4n", formatter.formatWithContext(formatter_context, stream_info)); + FormatterPtr formatter = *FormatterImpl::create(format, false); + EXPECT_EQ("%E4n", formatter->formatWithContext(formatter_context, stream_info)); } #endif } @@ -4814,22 +4790,22 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { const std::string format = "%PROTOCOL%|%RESP(not_exist)%|" "%REQ(FIRST?SECOND)%|%RESP(FIRST?SECOND)%|" "%TRAILER(THIRD)%|%TRAILER(TEST?TEST-2)%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(absl::nullopt)); - EXPECT_EQ("-|-|-|-|-|-", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("-|-|-|-|-|-", formatter->formatWithContext(formatter_context, stream_info)); } { const std::string format = "%PROTOCOL%|%RESP(not_exist)%|" "%REQ(FIRST?SECOND)%%RESP(FIRST?SECOND)%|" "%TRAILER(THIRD)%|%TRAILER(TEST?TEST-2)%"; - FormatterImpl formatter(format, true); + FormatterPtr formatter = *FormatterImpl::create(format, true); EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(absl::nullopt)); - EXPECT_EQ("||||", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("||||", formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4838,9 +4814,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { EXPECT_CALL(Const(stream_info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); const std::string format = "%DYNAMIC_METADATA(com.test:test_key)%|%DYNAMIC_METADATA(com.test:" "test_obj)%|%DYNAMIC_METADATA(com.test:test_obj:inner_key)%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); - EXPECT_EQ("-|-|-", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("-|-|-", formatter->formatWithContext(formatter_context, stream_info)); } { @@ -4849,27 +4825,27 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { EXPECT_CALL(Const(stream_info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); const std::string format = "%DYNAMIC_METADATA(com.test:test_key)%|%DYNAMIC_METADATA(com.test:" "test_obj)%|%DYNAMIC_METADATA(com.test:test_obj:inner_key)%"; - FormatterImpl formatter(format, true); + FormatterPtr formatter = *FormatterImpl::create(format, true); - EXPECT_EQ("||", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("||", formatter->formatWithContext(formatter_context, stream_info)); } { EXPECT_CALL(Const(stream_info), filterState()).Times(testing::AtLeast(1)); const std::string format = "%FILTER_STATE(testing)%|%FILTER_STATE(serialized)%|" "%FILTER_STATE(testing):8%|%FILTER_STATE(nonexisting)%"; - FormatterImpl formatter(format, false); + FormatterPtr formatter = *FormatterImpl::create(format, false); - EXPECT_EQ("-|-|-|-", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("-|-|-|-", formatter->formatWithContext(formatter_context, stream_info)); } { EXPECT_CALL(Const(stream_info), filterState()).Times(testing::AtLeast(1)); const std::string format = "%FILTER_STATE(testing)%|%FILTER_STATE(serialized)%|" "%FILTER_STATE(testing):8%|%FILTER_STATE(nonexisting)%"; - FormatterImpl formatter(format, true); + FormatterPtr formatter = *FormatterImpl::create(format, true); - EXPECT_EQ("|||", formatter.formatWithContext(formatter_context, stream_info)); + EXPECT_EQ("|||", formatter->formatWithContext(formatter_context, stream_info)); } } @@ -4919,7 +4895,10 @@ TEST(SubstitutionFormatterTest, ParserFailures) { "%START_TIME(%4On%)%"}; for (const std::string& test_case : test_cases) { - EXPECT_THROW(parser.parse(test_case), EnvoyException) << test_case; + EXPECT_THROW(THROW_OR_RETURN_VALUE(parser.parse(test_case), + std::vector>), + EnvoyException) + << test_case; } } @@ -4930,14 +4909,14 @@ TEST(SubstitutionFormatterTest, ParserSuccesses) { "%DOWNSTREAM_PEER_FINGERPRINT_256%"}; for (const std::string& test_case : test_cases) { - EXPECT_NO_THROW(parser.parse(test_case)); + EXPECT_TRUE(parser.parse(test_case).status().ok()); } } TEST(SubstitutionFormatterTest, EmptyFormatParse) { StreamInfo::MockStreamInfo stream_info; - auto providers = SubstitutionFormatParser::parse(""); + auto providers = *SubstitutionFormatParser::parse(""); EXPECT_EQ(providers.size(), 1); EXPECT_EQ("", providers[0]->formatWithContext({}, stream_info)); @@ -4946,7 +4925,7 @@ TEST(SubstitutionFormatterTest, EmptyFormatParse) { TEST(SubstitutionFormatterTest, EscapingFormatParse) { StreamInfo::MockStreamInfo stream_info; - auto providers = SubstitutionFormatParser::parse("%%"); + auto providers = *SubstitutionFormatParser::parse("%%"); ASSERT_EQ(providers.size(), 1); EXPECT_EQ("%", providers[0]->formatWithContext({}, stream_info)); @@ -4958,7 +4937,7 @@ TEST(SubstitutionFormatterTest, FormatterExtension) { std::vector commands; commands.push_back(std::make_unique()); - auto providers = SubstitutionFormatParser::parse("foo %COMMAND_EXTENSION(x)%", commands); + auto providers = *SubstitutionFormatParser::parse("foo %COMMAND_EXTENSION(x)%", commands); EXPECT_EQ(providers.size(), 2); EXPECT_EQ("TestFormatter", providers[1]->formatWithContext({}, stream_info)); @@ -4976,7 +4955,7 @@ TEST(SubstitutionFormatterTest, PercentEscapingEdgeCase) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - auto providers = SubstitutionFormatParser::parse("%HOSTNAME%%PROTOCOL%"); + auto providers = *SubstitutionFormatParser::parse("%HOSTNAME%%PROTOCOL%"); ASSERT_EQ(providers.size(), 2); EXPECT_EQ("myhostname", providers[0]->formatWithContext({}, stream_info)); @@ -4985,14 +4964,14 @@ TEST(SubstitutionFormatterTest, PercentEscapingEdgeCase) { TEST(SubstitutionFormatterTest, EnvironmentFormatterTest) { { - EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatParser::parse("%ENVIRONMENT()%"), EnvoyException, - "ENVIRONMENT requires parameters"); + EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatParser::parse("%ENVIRONMENT()%").IgnoreError(), + EnvoyException, "ENVIRONMENT requires parameters"); } { StreamInfo::MockStreamInfo stream_info; - auto providers = SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV)%"); + auto providers = *SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV)%"); ASSERT_EQ(providers.size(), 1); @@ -5005,7 +4984,7 @@ TEST(SubstitutionFormatterTest, EnvironmentFormatterTest) { TestEnvironment::setEnvVar("ENVOY_TEST_ENV", "test", 1); Envoy::Cleanup cleanup([]() { TestEnvironment::unsetEnvVar("ENVOY_TEST_ENV"); }); - auto providers = SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV)%"); + auto providers = *SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV)%"); ASSERT_EQ(providers.size(), 1); @@ -5018,7 +4997,7 @@ TEST(SubstitutionFormatterTest, EnvironmentFormatterTest) { TestEnvironment::setEnvVar("ENVOY_TEST_ENV", "test", 1); Envoy::Cleanup cleanup([]() { TestEnvironment::unsetEnvVar("ENVOY_TEST_ENV"); }); - auto providers = SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV):2%"); + auto providers = *SubstitutionFormatParser::parse("%ENVIRONMENT(ENVOY_TEST_ENV):2%"); ASSERT_EQ(providers.size(), 1); @@ -5086,7 +5065,7 @@ TEST(SubstitutionFormatterTest, UniqueIdFormatterTest) { StreamInfo::MockStreamInfo stream_info; // Simulate initial parsing of configuration - auto providers1 = SubstitutionFormatParser::parse("%UNIQUE_ID%"); + auto providers1 = *SubstitutionFormatParser::parse("%UNIQUE_ID%"); ASSERT_EQ(providers1.size(), 1); // Generate first unique ID with the initial configuration @@ -5101,7 +5080,7 @@ TEST(SubstitutionFormatterTest, UniqueIdFormatterTest) { EXPECT_NE(id1, id2); // Simulate configuration reload - auto providers2 = SubstitutionFormatParser::parse("%UNIQUE_ID%"); + auto providers2 = *SubstitutionFormatParser::parse("%UNIQUE_ID%"); ASSERT_EQ(providers2.size(), 1); // Generate another unique ID after the simulated reload diff --git a/test/common/http/conn_manager_impl_test_base.cc b/test/common/http/conn_manager_impl_test_base.cc index 0b70e2057938..bed8ebe01354 100644 --- a/test/common/http/conn_manager_impl_test_base.cc +++ b/test/common/http/conn_manager_impl_test_base.cc @@ -152,7 +152,7 @@ HttpConnectionManagerImplMixin::HttpConnectionManagerImplMixin() access_log_path_("dummy_path"), access_logs_{AccessLog::InstanceSharedPtr{new Extensions::AccessLoggers::File::FileAccessLog( Filesystem::FilePathAndType{Filesystem::DestinationType::File, access_log_path_}, {}, - Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), log_manager_)}}, + *Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), log_manager_)}}, codec_(new NiceMock()), stats_({ALL_HTTP_CONN_MAN_STATS(POOL_COUNTER(*fake_stats_.rootScope()), POOL_GAUGE(*fake_stats_.rootScope()), diff --git a/test/common/json/BUILD b/test/common/json/BUILD index 91bf779da9d6..8bbe61deb41c 100644 --- a/test/common/json/BUILD +++ b/test/common/json/BUILD @@ -91,6 +91,15 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "json_utility_test", + srcs = ["json_utility_test.cc"], + deps = [ + "//source/common/json:json_utility_lib", + "//source/common/protobuf:utility_lib", + ], +) + envoy_cc_test_binary( name = "gen_excluded_unicodes", srcs = ["gen_excluded_unicodes.cc"], diff --git a/test/common/json/json_utility_test.cc b/test/common/json/json_utility_test.cc new file mode 100644 index 000000000000..036efdd48734 --- /dev/null +++ b/test/common/json/json_utility_test.cc @@ -0,0 +1,88 @@ +#include "source/common/json/json_utility.h" +#include "source/common/protobuf/message_validator_impl.h" +#include "source/common/protobuf/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Json { +namespace { + +std::string toJson(const ProtobufWkt::Value& v) { + std::string json_string; + Utility::appendValueToString(v, json_string); + return json_string; +} + +TEST(JsonUtilityTest, AppendValueToString) { + ProtobufWkt::Value v; + + // null + EXPECT_EQ(toJson(v), "null"); + + v.set_null_value(ProtobufWkt::NULL_VALUE); + EXPECT_EQ(toJson(v), "null"); + + // bool + v.set_bool_value(true); + EXPECT_EQ(toJson(v), "true"); + + v.set_bool_value(false); + EXPECT_EQ(toJson(v), "false"); + + // number + v.set_number_value(1); + EXPECT_EQ(toJson(v), "1"); + + v.set_number_value(1.1); + EXPECT_EQ(toJson(v), "1.1"); + + // string + v.set_string_value("foo"); + EXPECT_EQ(toJson(v), "\"foo\""); + + // struct + auto* struct_value = v.mutable_struct_value(); + EXPECT_EQ(toJson(v), R"EOF({})EOF"); + + struct_value->mutable_fields()->insert({"foo", ValueUtil::stringValue("bar")}); + EXPECT_EQ(toJson(v), R"EOF({"foo":"bar"})EOF"); + + // list + auto* list_value = v.mutable_list_value(); + + EXPECT_EQ(toJson(v), R"EOF([])EOF"); + + list_value->add_values()->set_string_value("foo"); + list_value->add_values()->set_string_value("bar"); + + EXPECT_EQ(toJson(v), R"EOF(["foo","bar"])EOF"); + + // Complex structure + const std::string yaml = R"EOF( + a: + a: + - a: 1 + b: 2 + b: + - a: 3 + b: 4 + - a: 5 + c: true + d: [5, 3.14] + e: foo + f: 1.1 + b: [1, 2, 3] + c: bar + )EOF"; + + MessageUtil::loadFromYaml(yaml, v, ProtobufMessage::getNullValidationVisitor()); + + EXPECT_EQ( + toJson(v), + R"EOF({"a":{"a":[{"a":1,"b":2}],"b":[{"a":3,"b":4},{"a":5}],"c":true,"d":[5,"3.14"],"e":"foo","f":"1.1"},"b":[1,2,3],"c":"bar"})EOF"); +} + +} // namespace +} // namespace Json +} // namespace Envoy diff --git a/test/common/listener_manager/listener_manager_impl_quic_only_test.cc b/test/common/listener_manager/listener_manager_impl_quic_only_test.cc index 05dd2cf623bb..f088a12dd0ab 100644 --- a/test/common/listener_manager/listener_manager_impl_quic_only_test.cc +++ b/test/common/listener_manager/listener_manager_impl_quic_only_test.cc @@ -24,6 +24,21 @@ class MockSupportsUdpGso : public Api::OsSysCallsImpl { class ListenerManagerImplQuicOnlyTest : public ListenerManagerImplTest { public: + size_t expectedNumSocketOptions() { + // SO_REUSEPORT, IP_PKTINFO and IP_MTU_DISCOVER/IP_DONTFRAG. + const size_t num_platform_independent_socket_options = + Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment") ? 3 : 2; + size_t num_platform_dependent_socket_options = 0; +#ifdef SO_RXQ_OVFL + ++num_platform_dependent_socket_options; +#endif + if (Api::OsSysCallsSingleton::get().supportsUdpGro()) { + // SO_REUSEPORT + ++num_platform_dependent_socket_options; + } + return num_platform_dependent_socket_options + num_platform_independent_socket_options; + } + NiceMock udp_gso_syscall_; TestThreadsafeSingletonInjector os_calls{&udp_gso_syscall_}; Api::OsSysCallsImpl os_sys_calls_actual_; @@ -106,13 +121,7 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryAndSslContext) { .WillByDefault(Return(os_sys_calls_actual_.supportsUdpGso())); EXPECT_CALL(server_.api_.random_, uuid()); expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, -#ifdef SO_RXQ_OVFL // SO_REUSEPORT is on as configured - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 4 : 3, -#else - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 3 : 2, -#endif + expectedNumSocketOptions(), ListenerComponentFactory::BindType::ReusePort); expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, @@ -138,6 +147,18 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryAndSslContext) { } #endif +#ifdef ENVOY_IP_DONTFRAG + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_DONTFRAG, + /* expected_value */ 1, + /* expected_num_calls */ 1); +#else + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_MTU_DISCOVER, + /* expected_value */ IP_PMTUDISC_DO, + /* expected_num_calls */ 1); +#endif + addOrUpdateListener(listener_proto); EXPECT_EQ(1u, manager_->listeners().size()); EXPECT_FALSE(manager_->listeners()[0] @@ -195,13 +216,7 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicWriterFromConfig) { ON_CALL(udp_gso_syscall_, supportsUdpGso()).WillByDefault(Return(true)); EXPECT_CALL(server_.api_.random_, uuid()); expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, -#ifdef SO_RXQ_OVFL // SO_REUSEPORT is on as configured - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 4 : 3, -#else - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 3 : 2, -#endif + expectedNumSocketOptions(), ListenerComponentFactory::BindType::ReusePort); expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, @@ -227,6 +242,18 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicWriterFromConfig) { } #endif +#ifdef ENVOY_IP_DONTFRAG + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_DONTFRAG, + /* expected_value */ 1, + /* expected_num_calls */ 1); +#else + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_MTU_DISCOVER, + /* expected_value */ IP_PMTUDISC_DO, + /* expected_num_calls */ 1); +#endif + addOrUpdateListener(listener_proto); EXPECT_EQ(1u, manager_->listeners().size()); EXPECT_FALSE(manager_->listeners()[0] @@ -303,13 +330,7 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryWithExplictConnection ON_CALL(udp_gso_syscall_, supportsUdpGso()).WillByDefault(Return(true)); EXPECT_CALL(server_.api_.random_, uuid()); expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, -#ifdef SO_RXQ_OVFL // SO_REUSEPORT is on as configured - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 4 : 3, -#else - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 3 : 2, -#endif + expectedNumSocketOptions(), ListenerComponentFactory::BindType::ReusePort); expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, @@ -335,6 +356,18 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryWithExplictConnection } #endif +#ifdef ENVOY_IP_DONTFRAG + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_DONTFRAG, + /* expected_value */ 1, + /* expected_num_calls */ 1); +#else + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_MTU_DISCOVER, + /* expected_value */ IP_PMTUDISC_DO, + /* expected_num_calls */ 1); +#endif + addOrUpdateListener(listener_proto); EXPECT_EQ(1u, manager_->listeners().size()); EXPECT_FALSE(manager_->listeners()[0] @@ -359,13 +392,7 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFilterFromConfig) { ON_CALL(udp_gso_syscall_, supportsUdpGso()).WillByDefault(Return(true)); EXPECT_CALL(server_.api_.random_, uuid()); expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, -#ifdef SO_RXQ_OVFL // SO_REUSEPORT is on as configured - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 4 : 3, -#else - /* expected_num_options */ - Api::OsSysCallsSingleton::get().supportsUdpGro() ? 3 : 2, -#endif + expectedNumSocketOptions(), ListenerComponentFactory::BindType::ReusePort); expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, @@ -391,6 +418,18 @@ TEST_P(ListenerManagerImplQuicOnlyTest, QuicListenerFilterFromConfig) { } #endif +#ifdef ENVOY_IP_DONTFRAG + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_DONTFRAG, + /* expected_value */ 1, + /* expected_num_calls */ 1); +#else + expectSetsockopt(/* expected_sockopt_level */ IPPROTO_IP, + /* expected_sockopt_name */ IP_MTU_DISCOVER, + /* expected_value */ IP_PMTUDISC_DO, + /* expected_num_calls */ 1); +#endif + addOrUpdateListener(listener_proto); EXPECT_EQ(1u, manager_->listeners().size()); // Verify that the right filter chain type is installed. diff --git a/test/common/listener_manager/listener_manager_impl_test.cc b/test/common/listener_manager/listener_manager_impl_test.cc index d86093344ffc..cd7a7f658fe6 100644 --- a/test/common/listener_manager/listener_manager_impl_test.cc +++ b/test/common/listener_manager/listener_manager_impl_test.cc @@ -1008,11 +1008,12 @@ TEST_P(ListenerManagerImplTest, RejectTcpOptionsWithInternalListenerConfig) { for (const auto& f : listener_mutators) { auto new_listener = listener; f(new_listener); - EXPECT_THROW_WITH_MESSAGE(new ListenerImpl(new_listener, "version", *manager_, "foo", true, - false, /*hash=*/static_cast(0)), - EnvoyException, - "error adding listener named 'foo': has " - "unsupported tcp listener feature"); + EXPECT_EQ(ListenerImpl::create(new_listener, "version", *manager_, "foo", true, false, + /*hash=*/static_cast(0)) + .status() + .message(), + "error adding listener named 'foo': has " + "unsupported tcp listener feature"); } { auto new_listener = listener; @@ -7809,6 +7810,24 @@ TEST(ListenerMessageUtilTest, ListenerMessageHaveDifferentFilterChainsAreEquival EXPECT_TRUE(Server::ListenerMessageUtil::filterChainOnlyChange(listener1, listener2)); } +TEST_P(ListenerManagerImplForInPlaceFilterChainUpdateTest, InvalidAddress) { + // Worker is not started yet. + envoy::config::listener::v3::Listener listener_proto; + Protobuf::TextFormat::ParseFromString(R"EOF( + name: "foo" + address: { + socket_address: { + address: "127.0.0.1.0" + port_value: 1234 + } + } + filter_chains: {} + )EOF", + &listener_proto); + EXPECT_EQ(manager_->addOrUpdateListener(listener_proto, "", true).status().message(), + "malformed IP address: 127.0.0.1.0"); +} + TEST_P(ListenerManagerImplForInPlaceFilterChainUpdateTest, TraditionalUpdateIfWorkerNotStarted) { // Worker is not started yet. auto listener_proto = createDefaultListener(); @@ -8034,13 +8053,13 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, InvalidExtendConnectionBalanceCon extend_balance_config->mutable_typed_config()->set_type_url( "type.googleapis.com/google.protobuf.test"); - auto listener_impl = ListenerImpl(listener, "version", *manager_, "foo", true, false, - /*hash=*/static_cast(0)); + auto listener_impl = *ListenerImpl::create(listener, "version", *manager_, "foo", true, false, + /*hash=*/static_cast(0)); auto socket_factory = std::make_unique(); Network::Address::InstanceConstSharedPtr address( new Network::Address::Ipv4Instance("192.168.0.1", 80, nullptr)); EXPECT_CALL(*socket_factory, localAddress()).WillOnce(ReturnRef(address)); - EXPECT_EQ(listener_impl.addSocketFactory(std::move(socket_factory)).message(), + EXPECT_EQ(listener_impl->addSocketFactory(std::move(socket_factory)).message(), "Didn't find a registered implementation for type: 'google.protobuf.test'"); #endif } @@ -8051,13 +8070,13 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, EmptyConnectionBalanceConfig) { auto listener = createIPv4Listener("TCPListener"); listener.mutable_connection_balance_config(); - auto listener_impl = ListenerImpl(listener, "version", *manager_, "foo", true, false, - /*hash=*/static_cast(0)); + auto listener_impl = *ListenerImpl::create(listener, "version", *manager_, "foo", true, false, + /*hash=*/static_cast(0)); auto socket_factory = std::make_unique(); Network::Address::InstanceConstSharedPtr address( new Network::Address::Ipv4Instance("192.168.0.1", 80, nullptr)); EXPECT_CALL(*socket_factory, localAddress()).WillOnce(ReturnRef(address)); - EXPECT_EQ(listener_impl.addSocketFactory(std::move(socket_factory)).message(), + EXPECT_EQ(listener_impl->addSocketFactory(std::move(socket_factory)).message(), "No valid balance type for connection balance"); #endif } diff --git a/test/common/orca/orca_parser_test.cc b/test/common/orca/orca_parser_test.cc index 6b56f72d55aa..b86fcc7d31cc 100644 --- a/test/common/orca/orca_parser_test.cc +++ b/test/common/orca/orca_parser_test.cc @@ -1,4 +1,4 @@ -#include +#include #include "source/common/common/base64.h" #include "source/common/orca/orca_parser.h" @@ -7,12 +7,25 @@ #include "test/test_common/utility.h" #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "xds/data/orca/v3/orca_load_report.pb.h" namespace Envoy { namespace Orca { namespace { +const std::string formattedHeaderPrefixText() { + CONSTRUCT_ON_FIRST_USE(std::string, absl::StrCat(kHeaderFormatPrefixText, " ")); +} + +const std::string formattedHeaderPrefixJson() { + CONSTRUCT_ON_FIRST_USE(std::string, absl::StrCat(kHeaderFormatPrefixJson, " ")); +} + +const std::string formattedHeaderPrefixBin() { + CONSTRUCT_ON_FIRST_USE(std::string, absl::StrCat(kHeaderFormatPrefixBin, " ")); +} + // Returns an example OrcaLoadReport proto with all fields populated. static xds::data::orca::v3::OrcaLoadReport exampleOrcaLoadReport() { xds::data::orca::v3::OrcaLoadReport orca_load_report; @@ -42,7 +55,157 @@ TEST(OrcaParserUtilTest, MissingOrcaHeaders) { StatusHelpers::HasStatus(absl::NotFoundError("no ORCA data sent from the backend"))); } -TEST(OrcaParserUtilTest, BinaryHeader) { +TEST(OrcaParserUtilTest, InvalidOrcaHeaderPrefix) { + // Verify that error is returned when unknown/invalid prefix is found in ORCA + // header value. + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), "BAD random-value"}}; + EXPECT_THAT( + parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::InvalidArgumentError("unsupported ORCA header format: BAD"))); +} + +TEST(OrcaParserUtilTest, EmptyOrcaHeader) { + Http::TestRequestHeaderMapImpl headers{{std::string(kEndpointLoadMetricsHeader), ""}}; + EXPECT_THAT( + parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::InvalidArgumentError("unsupported ORCA header format: "))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeader) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), + "cpu_utilization:0.7,application_utilization:0.8,mem_utilization:0.9," + "rps_fractional:1000,eps:2," + "named_metrics.foo:123,named_metrics.bar:0.2")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::IsOkAndHolds(ProtoEq(exampleOrcaLoadReport()))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderIncorrectFieldType) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), "cpu_utilization:\"0.7\"")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus( + absl::InvalidArgumentError("unable to parse custom backend load metric " + "value(cpu_utilization): \"0.7\""))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderNanMetricValue) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), + "cpu_utilization:", std::numeric_limits::quiet_NaN())}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::InvalidArgumentError( + "custom backend load metric value(cpu_utilization) cannot be NaN."))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderInfinityMetricValue) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), + "cpu_utilization:", std::numeric_limits::infinity())}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::InvalidArgumentError( + "custom backend load metric value(cpu_utilization) cannot be " + "infinity."))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderContainsDuplicateMetric) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), "cpu_utilization:0.7,cpu_utilization:0.8")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::AlreadyExistsError(absl::StrCat( + kEndpointLoadMetricsHeader, " contains duplicate metric: cpu_utilization")))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderUnsupportedMetric) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), "cpu_utilization:0.7,unsupported_metric:0.8")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus( + absl::InvalidArgumentError("unsupported metric name: unsupported_metric"))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderContainsDuplicateNamedMetric) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat( + formattedHeaderPrefixText(), + "named_metrics.foo:123,named_metrics.duplicate:123,named_metrics.duplicate:0.2")}}; + EXPECT_THAT( + parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::AlreadyExistsError(absl::StrCat( + kEndpointLoadMetricsHeader, " contains duplicate metric: named_metrics.duplicate")))); +} + +TEST(OrcaParserUtilTest, NativeHttpEncodedHeaderContainsEmptyNamedMetricKey) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), "named_metrics.:123")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::InvalidArgumentError("named metric key is empty."))); +} + +TEST(OrcaParserUtilTest, InvalidNativeHttpEncodedHeader) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixText(), "not-a-list-of-key-value-pairs")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus( + absl::InvalidArgumentError("metric values cannot be empty strings"))); +} + +TEST(OrcaParserUtilTest, JsonHeader) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixJson(), + "{\"cpu_utilization\": 0.7, \"application_utilization\": 0.8, " + "\"mem_utilization\": 0.9, \"rps_fractional\": 1000, \"eps\": 2, " + "\"named_metrics\": {\"foo\": 123,\"bar\": 0.2}}")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::IsOkAndHolds(ProtoEq(exampleOrcaLoadReport()))); +} + +TEST(OrcaParserUtilTest, InvalidJsonHeader) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixJson(), "JSON not-a-valid-json-string")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::StatusCode::kInvalidArgument, + testing::HasSubstr("invalid JSON"))); +} + +TEST(OrcaParserUtilTest, JsonHeaderUnknownField) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixJson(), + "{\"cpu_utilization\": 0.7, \"application_utilization\": 0.8, " + "\"mem_utilization\": 0.9, \"rps_fractional\": 1000, \"eps\": 2, " + "\"unknown_field\": 2," + "\"named_metrics\": {\"foo\": 123,\"bar\": 0.2}}")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::StatusCode::kInvalidArgument, + testing::HasSubstr("invalid JSON"))); +} + +TEST(OrcaParserUtilTest, JsonHeaderIncorrectFieldType) { + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixJson(), "{\"cpu_utilization\": \"0.7\"")}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::HasStatus(absl::StatusCode::kInvalidArgument, + testing::HasSubstr("invalid JSON"))); +} + +TEST(OrcaParserUtilTest, LegacyBinaryHeader) { + // Verify processing of headers sent in legacy ORCA header inherited from gRPC + // implementation works as intended. const std::string proto_string = TestUtility::getProtobufBinaryStringFromMessage(exampleOrcaLoadReport()); const auto orca_load_report_header_bin = @@ -53,6 +216,20 @@ TEST(OrcaParserUtilTest, BinaryHeader) { StatusHelpers::IsOkAndHolds(ProtoEq(exampleOrcaLoadReport()))); } +TEST(OrcaParserUtilTest, BinaryHeader) { + // Verify serialized binary header processing when using default ORCA header + // and appropriate format prefix in the header value. + const std::string proto_string = + TestUtility::getProtobufBinaryStringFromMessage(exampleOrcaLoadReport()); + const auto orca_load_report_header_bin = + Envoy::Base64::encode(proto_string.c_str(), proto_string.length()); + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), + absl::StrCat(formattedHeaderPrefixBin(), orca_load_report_header_bin)}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::IsOkAndHolds(ProtoEq(exampleOrcaLoadReport()))); +} + TEST(OrcaParserUtilTest, InvalidBinaryHeader) { const std::string proto_string = TestUtility::getProtobufBinaryStringFromMessage(exampleOrcaLoadReport()); @@ -84,6 +261,20 @@ TEST(OrcaParserUtilTest, EmptyBinaryHeader) { testing::HasSubstr("ORCA binary header value is empty"))); } +TEST(OrcaParserUtilTest, BinHeaderPrecedence) { + // Verifies that the order of precedence (binary proto over native http + // format) is observed when multiple ORCA headers are sent from the backend. + const std::string proto_string = + TestUtility::getProtobufBinaryStringFromMessage(exampleOrcaLoadReport()); + const auto orca_load_report_header_bin = + Envoy::Base64::encode(proto_string.c_str(), proto_string.length()); + Http::TestRequestHeaderMapImpl headers{ + {std::string(kEndpointLoadMetricsHeader), "cpu_utilization:0.7"}, + {std::string(kEndpointLoadMetricsHeaderBin), orca_load_report_header_bin}}; + EXPECT_THAT(parseOrcaLoadReportHeaders(headers), + StatusHelpers::IsOkAndHolds(ProtoEq(exampleOrcaLoadReport()))); +} + } // namespace } // namespace Orca } // namespace Envoy diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD index 2f1c560192b5..36c0a8f1b57c 100644 --- a/test/common/quic/BUILD +++ b/test/common/quic/BUILD @@ -323,6 +323,7 @@ envoy_cc_test( "//test/mocks/upstream:cluster_info_mocks", "//test/mocks/upstream:transport_socket_match_mocks", "//test/test_common:test_runtime_lib", + "//test/test_common:threadsafe_singleton_injector_lib", ], ) diff --git a/test/common/quic/client_connection_factory_impl_test.cc b/test/common/quic/client_connection_factory_impl_test.cc index 5f505584b97b..7b407eb46f2f 100644 --- a/test/common/quic/client_connection_factory_impl_test.cc +++ b/test/common/quic/client_connection_factory_impl_test.cc @@ -14,6 +14,7 @@ #include "test/test_common/environment.h" #include "test/test_common/network_utility.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/threadsafe_singleton_injector.h" #include "quiche/quic/core/crypto/quic_client_session_cache.h" #include "quiche/quic/core/deterministic_connection_id_generator.h" @@ -134,6 +135,49 @@ TEST_P(QuicNetworkConnectionTest, SocketOptions) { client_connection->close(Network::ConnectionCloseType::NoFlush); } +TEST_P(QuicNetworkConnectionTest, PreBindSocketOptionsFailure) { + initialize(); + + auto socket_option = std::make_shared(); + auto socket_options = std::make_shared(); + socket_options->push_back(socket_option); + EXPECT_CALL(*socket_option, setOption(_, envoy::config::core::v3::SocketOption::STATE_PREBIND)) + .WillOnce(Return(false)); + + std::unique_ptr client_connection = createQuicNetworkConnection( + *quic_info_, crypto_config_, + quic::QuicServerId{factory_->clientContextConfig()->serverNameIndication(), PEER_PORT}, + dispatcher_, test_address_, test_address_, quic_stat_names_, {}, *store_.rootScope(), + socket_options, nullptr, connection_id_generator_, *factory_); + EnvoyQuicClientSession* session = static_cast(client_connection.get()); + session->Initialize(); + client_connection->connect(); + EXPECT_FALSE(session->connection()->connected()); + EXPECT_EQ(client_connection->state(), Network::Connection::State::Closed); +} + +TEST_P(QuicNetworkConnectionTest, PostBindSocketOptionsFailure) { + initialize(); + + auto socket_option = std::make_shared(); + auto socket_options = std::make_shared(); + socket_options->push_back(socket_option); + EXPECT_CALL(*socket_option, setOption(_, envoy::config::core::v3::SocketOption::STATE_PREBIND)); + EXPECT_CALL(*socket_option, setOption(_, envoy::config::core::v3::SocketOption::STATE_BOUND)) + .WillOnce(Return(false)); + + std::unique_ptr client_connection = createQuicNetworkConnection( + *quic_info_, crypto_config_, + quic::QuicServerId{factory_->clientContextConfig()->serverNameIndication(), PEER_PORT}, + dispatcher_, test_address_, test_address_, quic_stat_names_, {}, *store_.rootScope(), + socket_options, nullptr, connection_id_generator_, *factory_); + EnvoyQuicClientSession* session = static_cast(client_connection.get()); + session->Initialize(); + client_connection->connect(); + EXPECT_FALSE(session->connection()->connected()); + EXPECT_EQ(client_connection->state(), Network::Connection::State::Closed); +} + TEST_P(QuicNetworkConnectionTest, LocalAddress) { initialize(); Network::Address::InstanceConstSharedPtr local_addr = @@ -151,9 +195,41 @@ TEST_P(QuicNetworkConnectionTest, LocalAddress) { EXPECT_TRUE(client_connection->connecting()); EXPECT_EQ(Network::Connection::State::Open, client_connection->state()); EXPECT_THAT(client_connection->connectionInfoProvider().localAddress(), testing::NotNull()); + if (GetParam() == Network::Address::IpVersion::v6) { + EXPECT_TRUE(client_connection->connectionInfoProvider().localAddress()->ip()->ipv6()->v6only()); + } client_connection->close(Network::ConnectionCloseType::NoFlush); } +class MockGetSockOptSysCalls : public Api::OsSysCallsImpl { +public: + MOCK_METHOD(Api::SysCallIntResult, getsockopt, + (os_fd_t sockfd, int level, int optname, void* optval, socklen_t* optlen)); +}; + +TEST_P(QuicNetworkConnectionTest, GetV6OnlySocketOptionFailure) { + if (GetParam() == Network::Address::IpVersion::v4) { + return; + } + initialize(); + MockGetSockOptSysCalls os_sys_calls; + TestThreadsafeSingletonInjector singleton_injector{&os_sys_calls}; + + EXPECT_CALL(os_sys_calls, getsockopt(_, IPPROTO_IPV6, IPV6_V6ONLY, _, _)) + .WillOnce(Return(Api::SysCallIntResult{-1, SOCKET_ERROR_NOT_SUP})); + std::unique_ptr client_connection = createQuicNetworkConnection( + *quic_info_, crypto_config_, + quic::QuicServerId{factory_->clientContextConfig()->serverNameIndication(), PEER_PORT}, + dispatcher_, test_address_, test_address_, quic_stat_names_, {}, *store_.rootScope(), nullptr, + nullptr, connection_id_generator_, *factory_); + EnvoyQuicClientSession* session = static_cast(client_connection.get()); + session->Initialize(); + client_connection->connect(); + EXPECT_TRUE(client_connection->connecting()); + EXPECT_FALSE(session->connection()->connected()); + EXPECT_EQ(client_connection->state(), Network::Connection::State::Closed); +} + TEST_P(QuicNetworkConnectionTest, Srtt) { initialize(); diff --git a/test/common/quic/envoy_quic_client_session_test.cc b/test/common/quic/envoy_quic_client_session_test.cc index decd040b5bfe..8e100e5d77fe 100644 --- a/test/common/quic/envoy_quic_client_session_test.cc +++ b/test/common/quic/envoy_quic_client_session_test.cc @@ -676,8 +676,8 @@ TEST_P(EnvoyQuicClientSessionTest, WriteBlockedAndUnblock) { EnvoyQuicClientConnectionPeer::onFileEvent(*quic_connection_, Event::FileReadyType::Write, *quic_connection_->connectionSocket()); EXPECT_FALSE(quic_connection_->writer()->IsWriteBlocked()); - EXPECT_CALL(stream_callbacks, - onResetStream(Http::StreamResetReason::LocalReset, "QUIC_STREAM_REQUEST_REJECTED")); + EXPECT_CALL(stream_callbacks, onResetStream(Http::StreamResetReason::LocalReset, + "QUIC_STREAM_REQUEST_REJECTED|FROM_SELF")); EXPECT_CALL(*quic_connection_, SendControlFrame(_)); stream.resetStream(Http::StreamResetReason::LocalReset); } diff --git a/test/common/quic/envoy_quic_client_stream_test.cc b/test/common/quic/envoy_quic_client_stream_test.cc index e8fd682cadd6..6ae790c29d1d 100644 --- a/test/common/quic/envoy_quic_client_stream_test.cc +++ b/test/common/quic/envoy_quic_client_stream_test.cc @@ -210,6 +210,8 @@ TEST_F(EnvoyQuicClientStreamTest, GetRequestAndHeaderOnlyResponse) { const auto result = quic_stream_->encodeHeaders(request_headers_, /*end_stream=*/true); EXPECT_TRUE(result.ok()); + quic_stream_->setFlushTimeout(std::chrono::milliseconds(100)); // No-op + EXPECT_CALL(stream_decoder_, decodeHeaders_(_, /*end_stream=*/false)) .WillOnce(Invoke([](const Http::ResponseHeaderMapPtr& headers, bool) { EXPECT_EQ("200", headers->getStatusValue()); diff --git a/test/common/quic/envoy_quic_utils_test.cc b/test/common/quic/envoy_quic_utils_test.cc index 3fbfe54e07f4..c3ea2ff3c402 100644 --- a/test/common/quic/envoy_quic_utils_test.cc +++ b/test/common/quic/envoy_quic_utils_test.cc @@ -297,15 +297,58 @@ TEST(EnvoyQuicUtilsTest, CreateConnectionSocket) { EXPECT_TRUE(connection_socket->isOpen()); EXPECT_TRUE(connection_socket->ioHandle().wasConnected()); EXPECT_EQ("127.0.0.1", no_local_addr->ip()->addressAsString()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment")) { + int value = 0; + socklen_t val_length = sizeof(value); +#ifdef ENVOY_IP_DONTFRAG + RELEASE_ASSERT(connection_socket->getSocketOption(IPPROTO_IP, IP_DONTFRAG, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IP_DONTFRAG"); + EXPECT_EQ(value, 1); +#else + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IP, IP_MTU_DISCOVER, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IP_MTU_DISCOVER"); + EXPECT_EQ(value, IP_PMTUDISC_DO); +#endif + } connection_socket->close(); Network::Address::InstanceConstSharedPtr local_addr_v6 = - std::make_shared("::1"); + std::make_shared("::1", 0, nullptr, /*v6only*/ true); Network::Address::InstanceConstSharedPtr peer_addr_v6 = - std::make_shared("::1", 54321, nullptr); + std::make_shared("::1", 54321, nullptr, /*v6only*/ false); connection_socket = createConnectionSocket(peer_addr_v6, local_addr_v6, nullptr); EXPECT_TRUE(connection_socket->isOpen()); EXPECT_TRUE(connection_socket->ioHandle().wasConnected()); + EXPECT_TRUE(local_addr_v6->ip()->ipv6()->v6only()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment")) { + int value = 0; + socklen_t val_length = sizeof(value); +#ifdef ENVOY_IP_DONTFRAG + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IPV6, IPV6_DONTFRAG, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IPV6_DONTFRAG"); + ; + EXPECT_EQ(value, 1); +#else + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IPV6, IPV6_MTU_DISCOVER, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IPV6_MTU_DISCOVER"); + EXPECT_EQ(value, IPV6_PMTUDISC_DO); + // The v4 socket option is not applied to v6-only socket. + value = 0; + val_length = sizeof(value); + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IP, IP_MTU_DISCOVER, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IP_MTU_DISCOVER"); + EXPECT_NE(value, IP_PMTUDISC_DO); +#endif + } connection_socket->close(); Network::Address::InstanceConstSharedPtr no_local_addr_v6 = nullptr; @@ -313,6 +356,32 @@ TEST(EnvoyQuicUtilsTest, CreateConnectionSocket) { EXPECT_TRUE(connection_socket->isOpen()); EXPECT_TRUE(connection_socket->ioHandle().wasConnected()); EXPECT_EQ("::1", no_local_addr_v6->ip()->addressAsString()); + EXPECT_FALSE(no_local_addr_v6->ip()->ipv6()->v6only()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_set_do_not_fragment")) { + int value = 0; + socklen_t val_length = sizeof(value); +#ifdef ENVOY_IP_DONTFRAG + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IPV6, IPV6_DONTFRAG, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IPV6_DONTFRAG"); + EXPECT_EQ(value, 1); +#else + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IPV6, IPV6_MTU_DISCOVER, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IPV6_MTU_DISCOVER"); + EXPECT_EQ(value, IPV6_PMTUDISC_DO); + // The v4 socket option is also applied to dual stack socket. + value = 0; + val_length = sizeof(value); + RELEASE_ASSERT( + connection_socket->getSocketOption(IPPROTO_IP, IP_MTU_DISCOVER, &value, &val_length) + .return_value_ == 0, + "Failed getsockopt IP_MTU_DISCOVER"); + EXPECT_EQ(value, IP_PMTUDISC_DO); +#endif + } connection_socket->close(); } diff --git a/test/common/stream_info/bool_accessor_impl_test.cc b/test/common/stream_info/bool_accessor_impl_test.cc index dcc8c5ccc9f6..4f0845f05e65 100644 --- a/test/common/stream_info/bool_accessor_impl_test.cc +++ b/test/common/stream_info/bool_accessor_impl_test.cc @@ -22,6 +22,12 @@ TEST(BoolAccessorImplTest, TestProto) { EXPECT_NE(nullptr, message); } +TEST(BoolAccessorImplTest, TestString) { + BoolAccessorImpl accessor(true); + auto str = accessor.serializeAsString(); + EXPECT_EQ("true", str); +} + } // namespace } // namespace StreamInfo } // namespace Envoy diff --git a/test/common/stream_info/stream_info_impl_test.cc b/test/common/stream_info/stream_info_impl_test.cc index ca305401b108..6e8358c531ee 100644 --- a/test/common/stream_info/stream_info_impl_test.cc +++ b/test/common/stream_info/stream_info_impl_test.cc @@ -88,6 +88,10 @@ TEST_F(StreamInfoImplTest, TimingTest) { upstream_timing.onLastUpstreamRxByteReceived(test_time_.timeSystem()); dur = checkDuration(dur, timing.lastUpstreamRxByteReceived()); + EXPECT_FALSE(timing.lastDownstreamHeaderRxByteReceived()); + info.downstreamTiming().onLastDownstreamHeaderRxByteReceived(test_time_.timeSystem()); + dur = checkDuration(dur, timing.lastDownstreamHeaderRxByteReceived()); + EXPECT_FALSE(timing.firstDownstreamTxByteSent()); info.downstreamTiming().onFirstDownstreamTxByteSent(test_time_.timeSystem()); dur = checkDuration(dur, timing.firstDownstreamTxByteSent()); diff --git a/test/common/tls/context_impl_test.cc b/test/common/tls/context_impl_test.cc index 206514f17323..f1c4b926e1ca 100644 --- a/test/common/tls/context_impl_test.cc +++ b/test/common/tls/context_impl_test.cc @@ -1298,8 +1298,9 @@ TEST_F(ClientContextConfigImplTest, RSA2048Cert) { *tls_context.mutable_common_tls_context()->add_tls_certificates()); auto client_context_config = *ClientContextConfigImpl::create(tls_context, factory_context_); Stats::IsolatedStoreImpl store; - auto context = *manager_.createSslClientContext(*store.rootScope(), *client_context_config); - auto cleanup = cleanUpHelper(context); + auto context_or = manager_.createSslClientContext(*store.rootScope(), *client_context_config); + EXPECT_TRUE(context_or.ok()); + auto cleanup = cleanUpHelper(*context_or); } // Validate that 1024-bit RSA certificates are rejected. @@ -1370,8 +1371,9 @@ TEST_F(ClientContextConfigImplTest, RSA3072Cert) { auto client_context_config = *ClientContextConfigImpl::create(tls_context, factory_context_); ContextManagerImpl manager(server_factory_context_); Stats::IsolatedStoreImpl store; - auto context = *manager_.createSslClientContext(*store.rootScope(), *client_context_config); - auto cleanup = cleanUpHelper(context); + auto context_or = manager_.createSslClientContext(*store.rootScope(), *client_context_config); + EXPECT_TRUE(context_or.ok()); + auto cleanup = cleanUpHelper(*context_or); } // Validate that 4096-bit RSA certificates load successfully. @@ -1387,8 +1389,9 @@ TEST_F(ClientContextConfigImplTest, RSA4096Cert) { *tls_context.mutable_common_tls_context()->add_tls_certificates()); auto client_context_config = *ClientContextConfigImpl::create(tls_context, factory_context_); Stats::IsolatedStoreImpl store; - auto context = *manager_.createSslClientContext(*store.rootScope(), *client_context_config); - auto cleanup = cleanUpHelper(context); + auto context_or = manager_.createSslClientContext(*store.rootScope(), *client_context_config); + EXPECT_TRUE(context_or.ok()); + auto cleanup = cleanUpHelper(*context_or); } // Validate that P256 ECDSA certs load. @@ -1404,8 +1407,9 @@ TEST_F(ClientContextConfigImplTest, P256EcdsaCert) { *tls_context.mutable_common_tls_context()->add_tls_certificates()); auto client_context_config = *ClientContextConfigImpl::create(tls_context, factory_context_); Stats::IsolatedStoreImpl store; - auto context = *manager_.createSslClientContext(*store.rootScope(), *client_context_config); - auto cleanup = cleanUpHelper(context); + auto context_or = manager_.createSslClientContext(*store.rootScope(), *client_context_config); + EXPECT_TRUE(context_or.ok()); + auto cleanup = cleanUpHelper(*context_or); } // Validate that non-P256 ECDSA certs are rejected. diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 38917ad1be96..fc78e4c7189d 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -5594,6 +5594,69 @@ TEST_F(ClusterInfoImplTest, Http2AutoWithNonAlpnMatcherAndValidationOff) { EXPECT_NO_THROW(makeCluster(yaml + auto_http2)); } +TEST_F(ClusterInfoImplTest, MaxResponseHeadersDefault) { + const std::string yaml = TestEnvironment::substitute(R"EOF( + name: name + connect_timeout: 0.25s + type: STRICT_DNS + )EOF", + Network::Address::IpVersion::v4); + + auto cluster = makeCluster(yaml); + EXPECT_FALSE(cluster->info()->maxResponseHeadersKb().has_value()); + EXPECT_EQ(100, cluster->info()->maxResponseHeadersCount()); +} + +// Test that the runtime override for the defaults is used when specified. +TEST_F(ClusterInfoImplTest, MaxResponseHeadersRuntimeOverride) { + const std::string yaml = TestEnvironment::substitute(R"EOF( + name: name + connect_timeout: 0.25s + type: STRICT_DNS + )EOF", + Network::Address::IpVersion::v4); + + EXPECT_CALL(runtime_.snapshot_, + getInteger("envoy.reloadable_features.max_response_headers_size_kb", _)) + .WillRepeatedly(Return(123)); + EXPECT_CALL(runtime_.snapshot_, + getInteger("envoy.reloadable_features.max_response_headers_count", _)) + .WillRepeatedly(Return(456)); + + auto cluster = makeCluster(yaml); + EXPECT_EQ(absl::make_optional(uint16_t(123)), cluster->info()->maxResponseHeadersKb()); + EXPECT_EQ(456, cluster->info()->maxResponseHeadersCount()); +} + +// Test that the runtime override is ignored if there is a configured value. +TEST_F(ClusterInfoImplTest, MaxResponseHeadersRuntimeOverrideIgnored) { + const std::string yaml = TestEnvironment::substitute(R"EOF( + name: name + connect_timeout: 0.25s + type: STRICT_DNS + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + common_http_protocol_options: + max_response_headers_kb: 1 + max_headers_count: 2 + explicit_http_config: + http2_protocol_options: {} + )EOF", + Network::Address::IpVersion::v4); + + EXPECT_CALL(runtime_.snapshot_, + getDouble("envoy.reloadable_features.max_response_headers_size_kb", _)) + .WillRepeatedly(Return(123)); + EXPECT_CALL(runtime_.snapshot_, + getDouble("envoy.reloadable_features.max_response_headers_count", _)) + .WillRepeatedly(Return(456)); + + auto cluster = makeCluster(yaml); + EXPECT_EQ(absl::make_optional(uint16_t(1)), cluster->info()->maxResponseHeadersKb()); + EXPECT_EQ(2, cluster->info()->maxResponseHeadersCount()); +} + TEST_F(ClusterInfoImplTest, UpstreamFilterTypedAndDynamicConfigThrows) { const std::string yaml = R"EOF( name: name diff --git a/test/extensions/clusters/original_dst/original_dst_cluster_test.cc b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc index d0e1cd8e82f7..ba0777efa77b 100644 --- a/test/extensions/clusters/original_dst/original_dst_cluster_test.cc +++ b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc @@ -1163,10 +1163,8 @@ TEST(DestinationAddress, ObjectFactory) { auto object = factory->createFromBytes(address); ASSERT_NE(nullptr, object); EXPECT_EQ(address, object->serializeAsString()); - auto mirror = factory->reflect(object.get()); - ASSERT_NE(nullptr, mirror); - EXPECT_THAT(mirror->getField("ip"), testing::VariantWith("10.0.0.10")); - EXPECT_THAT(mirror->getField("port"), testing::VariantWith(8080)); + EXPECT_THAT(object->getField("ip"), testing::VariantWith("10.0.0.10")); + EXPECT_THAT(object->getField("port"), testing::VariantWith(8080)); EXPECT_EQ(nullptr, factory->createFromBytes("foo")); } diff --git a/test/extensions/config_subscription/grpc/BUILD b/test/extensions/config_subscription/grpc/BUILD index b0f9ce75ed79..0b41fb7b9056 100644 --- a/test/extensions/config_subscription/grpc/BUILD +++ b/test/extensions/config_subscription/grpc/BUILD @@ -32,6 +32,7 @@ envoy_cc_test( "//test/test_common:logging_lib", "//test/test_common:resources_lib", "//test/test_common:simulated_time_system_lib", + "//test/test_common:status_utility_lib", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", @@ -64,6 +65,7 @@ envoy_cc_test( "//test/test_common:logging_lib", "//test/test_common:resources_lib", "//test/test_common:simulated_time_system_lib", + "//test/test_common:status_utility_lib", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", @@ -180,6 +182,7 @@ envoy_cc_test( "//test/test_common:logging_lib", "//test/test_common:resources_lib", "//test/test_common:simulated_time_system_lib", + "//test/test_common:status_utility_lib", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/api/v2:pkg_cc_proto", @@ -270,6 +273,7 @@ envoy_cc_test( "//test/common/stats:stat_test_utility_lib", "//test/mocks/config:config_mocks", "//test/mocks/event:event_mocks", + "//test/test_common:test_runtime_lib", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/config_subscription/grpc/grpc_mux_failover_test.cc b/test/extensions/config_subscription/grpc/grpc_mux_failover_test.cc index 8a990f134fcb..dae34fd62ec8 100644 --- a/test/extensions/config_subscription/grpc/grpc_mux_failover_test.cc +++ b/test/extensions/config_subscription/grpc/grpc_mux_failover_test.cc @@ -6,6 +6,7 @@ #include "test/extensions/config_subscription/grpc/mocks.h" #include "test/mocks/config/mocks.h" #include "test/mocks/event/mocks.h" +#include "test/test_common/test_runtime.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -15,7 +16,6 @@ using testing::Return; namespace Envoy { namespace Config { -namespace { // Validates that if no failover is set, then all actions are essentially a pass // through. @@ -237,6 +237,13 @@ class GrpcMuxFailoverTest : public testing::Test { failover_callbacks_->onDiscoveryResponse(std::move(response), cp_stats); } + void invokeCloseStream() { + // A wrapper that invokes closeStream(). It is needed because closeStream() + // is a private method, and while this class is a friend for GrpcMuxFailover, + // the tests cannot invoke the method directly. + grpc_mux_failover_->closeStream(); + } + // Override a timer to emulate its expiration without waiting for it to expire. NiceMock dispatcher_; Event::MockTimer* timer_; @@ -440,6 +447,34 @@ TEST_F(GrpcMuxFailoverTest, AlternatingPrimaryAndFailoverAttemptsAfterFailoverAv grpc_mux_failover_->establishNewStream(); } +// Validation that when envoy.reloadable_features.xds_failover_to_primary_enabled is disabled +// and after the failover is available (a response is received), Envoy will only +// try to reconnect to the failover. +// This test will be removed once envoy.reloadable_features.xds_failover_to_primary_enabled +// is deprecated. +TEST_F(GrpcMuxFailoverTest, StickToFailoverAfterFailoverAvailable) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.xds_failover_to_primary_enabled", "false"}}); + connectToFailover(); + + // Emulate 5 disconnects, and ensure the primary reconnection isn't attempted. + for (int attempt = 0; attempt < 5; ++attempt) { + // Emulate a failover source failure that will not result in an attempt to + // connect to the primary. It should not close the failover stream (so + // the retry mechanism will kick in). + EXPECT_CALL(failover_stream_, closeStream()).Times(0); + EXPECT_CALL(grpc_mux_callbacks_, onEstablishmentFailure(true)); + EXPECT_CALL(primary_stream_, establishNewStream()).Times(0); + failover_callbacks_->onEstablishmentFailure(true); + } + + // Emulate a call to establishNewStream() of the failover stream. + EXPECT_CALL(primary_stream_, establishNewStream()).Times(0); + EXPECT_CALL(failover_stream_, establishNewStream()); + grpc_mux_failover_->establishNewStream(); +} + // Validates that multiple calls to establishNewStream when connecting to the // failover are invoked on the failover stream, and not the primary. TEST_F(GrpcMuxFailoverTest, MultipleEstablishFailoverStream) { @@ -626,26 +661,38 @@ TEST_F(GrpcMuxFailoverTest, OnWriteableConnectedToPrimaryInvoked) { // Validates that when connected to primary, a subsequent call to establishNewStream // will not try to recreate the stream. TEST_F(GrpcMuxFailoverTest, NoRecreateStreamWhenConnectedToPrimary) { - // Validate connected to primary. - { - connectToPrimary(); - EXPECT_CALL(primary_stream_, establishNewStream()).Times(0); - EXPECT_CALL(failover_stream_, establishNewStream()).Times(0); - grpc_mux_failover_->establishNewStream(); - } + connectToPrimary(); + EXPECT_CALL(primary_stream_, establishNewStream()).Times(0); + EXPECT_CALL(failover_stream_, establishNewStream()).Times(0); + grpc_mux_failover_->establishNewStream(); } // Validates that when connected to failover, a subsequent call to establishNewStream // will not try to recreate the stream. TEST_F(GrpcMuxFailoverTest, NoRecreateStreamWhenConnectedToFailover) { - // Validate connected to failover. - { - connectToFailover(); - EXPECT_CALL(primary_stream_, establishNewStream()).Times(0); - EXPECT_CALL(failover_stream_, establishNewStream()).Times(0); - grpc_mux_failover_->establishNewStream(); - } + connectToFailover(); + EXPECT_CALL(primary_stream_, establishNewStream()).Times(0); + EXPECT_CALL(failover_stream_, establishNewStream()).Times(0); + grpc_mux_failover_->establishNewStream(); } -} // namespace + +// Validates that closing the stream when connected to primary closes the +// primary stream. +TEST_F(GrpcMuxFailoverTest, CloseStreamWhenConnectedToPrimary) { + connectToPrimary(); + EXPECT_CALL(primary_stream_, closeStream()); + EXPECT_CALL(failover_stream_, closeStream()).Times(0); + invokeCloseStream(); +} + +// Validates that closing the stream when connected to failover closes the +// failover stream. +TEST_F(GrpcMuxFailoverTest, CloseStreamWhenConnectedToFailover) { + connectToFailover(); + EXPECT_CALL(primary_stream_, closeStream()).Times(0); + EXPECT_CALL(failover_stream_, closeStream()); + invokeCloseStream(); +} + } // namespace Config } // namespace Envoy diff --git a/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc index df541cd6bf98..5108a0502afd 100644 --- a/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc @@ -27,6 +27,7 @@ #include "test/test_common/logging.h" #include "test/test_common/resources.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/status_utility.h" #include "test/test_common/test_runtime.h" #include "test/test_common/test_time.h" #include "test/test_common/utility.h" @@ -95,7 +96,8 @@ class GrpcMuxImplTestBase : public testing::TestWithParam { const std::vector& resource_names, const std::string& version, bool first = false, const std::string& nonce = "", const Protobuf::int32 error_code = Grpc::Status::WellKnownGrpcStatus::Ok, - const std::string& error_message = "") { + const std::string& error_message = "", + Grpc::MockAsyncStream* async_stream = nullptr) { envoy::service::discovery::v3::DiscoveryRequest expected_request; if (first) { expected_request.mutable_node()->CopyFrom(local_info_.node()); @@ -113,7 +115,8 @@ class GrpcMuxImplTestBase : public testing::TestWithParam { error_detail->set_code(error_code); error_detail->set_message(error_message); } - EXPECT_CALL(async_stream_, sendMessageRaw_(Grpc::ProtoBufferEq(expected_request), false)); + EXPECT_CALL(async_stream ? *async_stream : async_stream_, + sendMessageRaw_(Grpc::ProtoBufferEq(expected_request), false)); } TestScopedRuntime scoped_runtime_; @@ -122,6 +125,9 @@ class GrpcMuxImplTestBase : public testing::TestWithParam { NiceMock local_info_; Grpc::MockAsyncClient* async_client_; Grpc::MockAsyncStream async_stream_; + // Used for tests invoking updateMuxSource(). + Grpc::MockAsyncClient* replaced_async_client_; + Grpc::MockAsyncStream replaced_async_stream_; CustomConfigValidatorsPtr config_validators_; GrpcMuxImplPtr grpc_mux_; NiceMock callbacks_; @@ -1346,6 +1352,172 @@ TEST_P(GrpcMuxImplTest, RemoveCachedResourceOnLastSubscription) { EXPECT_CALL(*eds_resources_cache_, removeResource("x")); } +// Updating the mux object while being connected sends the correct requests. +TEST_P(GrpcMuxImplTest, MuxDynamicReplacementWhenConnected) { + replaced_async_client_ = new Grpc::MockAsyncClient(); + setup(); + InSequence s; + + auto foo_sub = grpc_mux_->addWatch("type_url_foo", {"x", "y"}, callbacks_, resource_decoder_, {}); + auto bar_sub = grpc_mux_->addWatch("type_url_bar", {}, callbacks_, resource_decoder_, {}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + expectSendMessage("type_url_bar", {}, ""); + grpc_mux_->start(); + EXPECT_EQ(1, control_plane_connected_state_.value()); + + // Switch the mux. + envoy::config::core::v3::ApiConfigSource empty_ads_config; + // Expect a disconnect from the original async_client and stream. + EXPECT_CALL(async_stream_, resetStream()); + // Expect establishing connection to the new client and stream. + EXPECT_CALL(*replaced_async_client_, startRaw(_, _, _, _)) + .WillOnce(Return(&replaced_async_stream_)); + // Expect the initial messages to be sent to the new stream. + expectSendMessage("type_url_foo", {"x", "y"}, "", true, "", Grpc::Status::WellKnownGrpcStatus::Ok, + "", &replaced_async_stream_); + expectSendMessage("type_url_bar", {}, "", false, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + &replaced_async_stream_); + EXPECT_OK(grpc_mux_->updateMuxSource( + /*primary_async_client=*/std::unique_ptr(replaced_async_client_), + /*failover_async_client=*/nullptr, + /*custom_config_validators=*/nullptr, + /*scope=*/*stats_.rootScope(), + /*backoff_strategy=*/ + std::make_unique( + SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_), + empty_ads_config)); + // Ending test, removing subscriptions for type_url_foo. + expectSendMessage("type_url_foo", {}, "", false, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + &replaced_async_stream_); +} + +// Updating the mux object after receiving a response, sends the correct requests. +TEST_P(GrpcMuxImplTest, MuxDynamicReplacementFetchingResources) { + replaced_async_client_ = new Grpc::MockAsyncClient(); + setup(); + InSequence s; + + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + OpaqueResourceDecoderSharedPtr resource_decoder( + std::make_shared>("cluster_name")); + auto foo_sub = grpc_mux_->addWatch(type_url, {"x", "y"}, callbacks_, resource_decoder, {}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {"x", "y"}, "", true); + grpc_mux_->start(); + + // Send back a response for one of the resources. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([&load_assignment](const std::vector& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + const auto& expected_assignment = + dynamic_cast( + resources[0].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + return absl::OkStatus(); + })); + expectSendMessage(type_url, {"x", "y"}, "1"); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + } + + // Switch the mux. + envoy::config::core::v3::ApiConfigSource empty_ads_config; + // Expect a disconnect from the original async_client and stream. + EXPECT_CALL(async_stream_, resetStream()); + // Expect establishing connection to the new client and stream. + EXPECT_CALL(*replaced_async_client_, startRaw(_, _, _, _)) + .WillOnce(Return(&replaced_async_stream_)); + // Expect the initial message to be sent to the new stream. + expectSendMessage(type_url, {"x", "y"}, "1", true, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + &replaced_async_stream_); + EXPECT_OK(grpc_mux_->updateMuxSource( + /*primary_async_client=*/std::unique_ptr(replaced_async_client_), + /*failover_async_client=*/nullptr, + /*custom_config_validators=*/std::make_unique>(), + /*scope=*/*stats_.rootScope(), + /*backoff_strategy=*/ + std::make_unique( + SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_), + empty_ads_config)); + + // Send a response to resource "y" on the replaced mux. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("2"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("y"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "2")) + .WillOnce(Invoke([&load_assignment](const std::vector& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + const auto& expected_assignment = + dynamic_cast( + resources[0].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + return absl::OkStatus(); + })); + expectSendMessage(type_url, {"x", "y"}, "2", false, "", Grpc::Status::WellKnownGrpcStatus::Ok, + "", &replaced_async_stream_); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + } + + // Ending test, removing subscriptions for the subscription. + expectSendMessage(type_url, {}, "2", false, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + &replaced_async_stream_); +} + +// Updating the mux object with wrong rate limit settings is rejected. +TEST_P(GrpcMuxImplTest, RejectMuxDynamicReplacementRateLimitSettingsError) { + replaced_async_client_ = new Grpc::MockAsyncClient(); + setup(); + InSequence s; + + auto foo_sub = grpc_mux_->addWatch("type_url_foo", {"x", "y"}, callbacks_, resource_decoder_, {}); + auto bar_sub = grpc_mux_->addWatch("type_url_bar", {}, callbacks_, resource_decoder_, {}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + expectSendMessage("type_url_bar", {}, ""); + grpc_mux_->start(); + EXPECT_EQ(1, control_plane_connected_state_.value()); + + // Switch the mux. + envoy::config::core::v3::ApiConfigSource ads_config_wrong_settings; + envoy::config::core::v3::RateLimitSettings* rate_limits = + ads_config_wrong_settings.mutable_rate_limit_settings(); + rate_limits->mutable_max_tokens()->set_value(500); + rate_limits->mutable_fill_rate()->set_value(std::numeric_limits::quiet_NaN()); + // No disconnect and replacement of the original async_client. + EXPECT_CALL(async_stream_, resetStream()).Times(0); + EXPECT_CALL(*replaced_async_client_, startRaw(_, _, _, _)).Times(0); + EXPECT_FALSE(grpc_mux_ + ->updateMuxSource( + /*primary_async_client=*/std::unique_ptr( + replaced_async_client_), + /*failover_async_client=*/nullptr, + /*custom_config_validators=*/nullptr, + /*scope=*/*stats_.rootScope(), + /*backoff_strategy=*/ + std::make_unique( + SubscriptionFactory::RetryInitialDelayMs, + SubscriptionFactory::RetryMaxDelayMs, random_), + ads_config_wrong_settings) + .ok()); + // Ending test, removing subscriptions for type_url_foo. + expectSendMessage("type_url_foo", {}, "", false, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + &async_stream_); +} + /** * Tests the NullGrpcMuxImpl object to increase code-coverage. */ @@ -1399,6 +1571,14 @@ TEST_F(NullGrpcMuxImplTest, OnDiscoveryResponseImplemented) { EXPECT_NO_THROW(null_mux_.onDiscoveryResponse(std::move(response), cp_stats)); } +TEST_F(NullGrpcMuxImplTest, UpdateMuxSourceError) { + Stats::TestUtil::TestStore stats; + const envoy::config::core::v3::ApiConfigSource empty_config; + const absl::Status status = null_mux_.updateMuxSource(nullptr, nullptr, nullptr, + *stats.rootScope(), nullptr, empty_config); + EXPECT_EQ(status.code(), absl::StatusCode::kUnimplemented); +} + TEST(GrpcMuxFactoryTest, InvalidRateLimit) { auto* factory = Config::Utility::getFactoryByName("envoy.config_mux.grpc_mux_factory"); diff --git a/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc index 5c27473e03f0..b787557025c3 100644 --- a/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc @@ -28,6 +28,7 @@ #include "test/test_common/logging.h" #include "test/test_common/resources.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/status_utility.h" #include "test/test_common/test_runtime.h" #include "test/test_common/test_time.h" #include "test/test_common/utility.h" @@ -101,7 +102,8 @@ class NewGrpcMuxImplTestBase : public testing::TestWithParam& initial_resource_versions = {}) { + const std::map& initial_resource_versions = {}, + Grpc::MockAsyncStream* async_stream = nullptr) { API_NO_BOOST(envoy::service::discovery::v3::DeltaDiscoveryRequest) expected_request; expected_request.mutable_node()->CopyFrom(local_info_.node()); for (const auto& resource : resource_names_subscribe) { @@ -120,7 +122,8 @@ class NewGrpcMuxImplTestBase : public testing::TestWithParamset_code(error_code); error_detail->set_message(error_message); } - EXPECT_CALL(async_stream_, sendMessageRaw_(Grpc::ProtoBufferEq(expected_request), false)); + EXPECT_CALL(async_stream ? *async_stream : async_stream_, + sendMessageRaw_(Grpc::ProtoBufferEq(expected_request), false)); } void remoteClose() { @@ -176,6 +179,9 @@ class NewGrpcMuxImplTestBase : public testing::TestWithParam random_; Grpc::MockAsyncClient* async_client_; NiceMock async_stream_; + // Used for tests invoking updateMuxSource(). + Grpc::MockAsyncClient* replaced_async_client_; + Grpc::MockAsyncStream replaced_async_stream_; CustomConfigValidatorsPtr config_validators_; NiceMock local_info_; std::unique_ptr grpc_mux_; @@ -795,6 +801,175 @@ TEST_P(NewGrpcMuxImplTest, AddRemoveSubscriptions) { } } +// Updating the mux object while being connected sends the correct requests. +TEST_P(NewGrpcMuxImplTest, MuxDynamicReplacementWhenConnected) { + replaced_async_client_ = new Grpc::MockAsyncClient(); + setup(); + InSequence s; + + auto foo_sub = grpc_mux_->addWatch("type_url_foo", {"x", "y"}, callbacks_, resource_decoder_, {}); + auto bar_sub = grpc_mux_->addWatch("type_url_bar", {}, callbacks_, resource_decoder_, {}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage("type_url_foo", {"x", "y"}, {}); + expectSendMessage("type_url_bar", {}, {}); + grpc_mux_->start(); + EXPECT_EQ(1, control_plane_connected_state_.value()); + + // Switch the mux. + envoy::config::core::v3::ApiConfigSource empty_ads_config; + // Expect a disconnect from the original async_client and stream. + EXPECT_CALL(async_stream_, resetStream()); + // Expect establishing connection to the new client and stream. + EXPECT_CALL(*replaced_async_client_, startRaw(_, _, _, _)) + .WillOnce(Return(&replaced_async_stream_)); + // Expect the initial messages to be sent to the new stream. + expectSendMessage("type_url_foo", {"x", "y"}, {}, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + {}, &replaced_async_stream_); + expectSendMessage("type_url_bar", {}, {}, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", {}, + &replaced_async_stream_); + EXPECT_OK(grpc_mux_->updateMuxSource( + /*primary_async_client=*/std::unique_ptr(replaced_async_client_), + /*failover_async_client=*/nullptr, + /*custom_config_validators=*/nullptr, + /*scope=*/*stats_.rootScope(), + /*backoff_strategy=*/ + std::make_unique( + SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_), + empty_ads_config)); + // Ending test, removing subscriptions for type_url_foo. + expectSendMessage("type_url_foo", {}, {"x", "y"}, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + {}, &replaced_async_stream_); +} + +// Updating the mux object after receiving a response, sends the correct requests. +TEST_P(NewGrpcMuxImplTest, MuxDynamicReplacementFetchingResources) { + replaced_async_client_ = new Grpc::MockAsyncClient(); + setup(); + InSequence s; + + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + auto foo_sub = grpc_mux_->addWatch(type_url, {"x", "y"}, callbacks_, resource_decoder_, {}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {"x", "y"}, {}); + grpc_mux_->start(); + + // Send back a response for one of the resources. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_system_version_info("1"); + response->set_nonce("n1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + auto* resource = response->add_resources(); + resource->set_name("x"); + resource->mutable_resource()->PackFrom(load_assignment); + resource->set_version("x1"); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "1")) + .WillOnce(Invoke([&load_assignment](const std::vector& added_resources, + const Protobuf::RepeatedPtrField&, + const std::string&) { + EXPECT_EQ(1, added_resources.size()); + EXPECT_TRUE( + TestUtility::protoEqual(added_resources[0].get().resource(), load_assignment)); + return absl::OkStatus(); + })); + expectSendMessage(type_url, {}, {}, "n1"); + onDiscoveryResponse(std::move(response)); + } + + // Switch the mux. + envoy::config::core::v3::ApiConfigSource empty_ads_config; + // Expect a disconnect from the original async_client and stream. + EXPECT_CALL(async_stream_, resetStream()); + // Expect establishing connection to the new client and stream. + EXPECT_CALL(*replaced_async_client_, startRaw(_, _, _, _)) + .WillOnce(Return(&replaced_async_stream_)); + // Expect the initial message to be sent to the new stream. + // It will include "x" in its initial_resource_versions. + expectSendMessage(type_url, {"x", "y"}, {}, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + {{"x", "x1"}}, &replaced_async_stream_); + EXPECT_OK(grpc_mux_->updateMuxSource( + /*primary_async_client=*/std::unique_ptr(replaced_async_client_), + /*failover_async_client=*/nullptr, + /*custom_config_validators=*/std::make_unique>(), + /*scope=*/*stats_.rootScope(), + /*backoff_strategy=*/ + std::make_unique( + SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_), + empty_ads_config)); + + // Send a response to resource "y" on the replaced mux. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_system_version_info("2"); + response->set_nonce("n2"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("y"); + auto* resource = response->add_resources(); + resource->set_name("y"); + resource->mutable_resource()->PackFrom(load_assignment); + resource->set_version("y1"); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "2")) + .WillOnce(Invoke([&load_assignment](const std::vector& added_resources, + const Protobuf::RepeatedPtrField&, + const std::string&) { + EXPECT_EQ(1, added_resources.size()); + EXPECT_TRUE( + TestUtility::protoEqual(added_resources[0].get().resource(), load_assignment)); + return absl::OkStatus(); + })); + expectSendMessage(type_url, {}, {}, "n2", Grpc::Status::WellKnownGrpcStatus::Ok, "", {}, + &replaced_async_stream_); + onDiscoveryResponse(std::move(response)); + } + + // Ending test, removing subscriptions for the subscription. + expectSendMessage(type_url, {}, {"x", "y"}, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", {}, + &replaced_async_stream_); +} + +// Updating the mux object with wrong rate limit settings is rejected. +TEST_P(NewGrpcMuxImplTest, RejectMuxDynamicReplacementRateLimitSettingsError) { + replaced_async_client_ = new Grpc::MockAsyncClient(); + setup(); + InSequence s; + + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + auto foo_sub = grpc_mux_->addWatch(type_url, {"x", "y"}, callbacks_, resource_decoder_, {}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {"x", "y"}, {}); + grpc_mux_->start(); + EXPECT_EQ(1, control_plane_connected_state_.value()); + + // Switch the mux. + envoy::config::core::v3::ApiConfigSource ads_config_wrong_settings; + envoy::config::core::v3::RateLimitSettings* rate_limits = + ads_config_wrong_settings.mutable_rate_limit_settings(); + rate_limits->mutable_max_tokens()->set_value(500); + rate_limits->mutable_fill_rate()->set_value(std::numeric_limits::quiet_NaN()); + // No disconnect and replacement of the original async_client. + EXPECT_CALL(async_stream_, resetStream()).Times(0); + EXPECT_CALL(*replaced_async_client_, startRaw(_, _, _, _)).Times(0); + EXPECT_FALSE(grpc_mux_ + ->updateMuxSource( + /*primary_async_client=*/std::unique_ptr( + replaced_async_client_), + /*failover_async_client=*/nullptr, + /*custom_config_validators=*/nullptr, + /*scope=*/*stats_.rootScope(), + /*backoff_strategy=*/ + std::make_unique( + SubscriptionFactory::RetryInitialDelayMs, + SubscriptionFactory::RetryMaxDelayMs, random_), + ads_config_wrong_settings) + .ok()); + // Ending test, removing subscriptions for type_url_foo. + expectSendMessage(type_url, {}, {"x", "y"}, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", {}, + &async_stream_); +} + TEST(NewGrpcMuxFactoryTest, InvalidRateLimit) { auto* factory = Config::Utility::getFactoryByName( "envoy.config_mux.new_grpc_mux_factory"); diff --git a/test/extensions/config_subscription/grpc/watch_map_test.cc b/test/extensions/config_subscription/grpc/watch_map_test.cc index 4186b3d8b8d7..014a1221b0c4 100644 --- a/test/extensions/config_subscription/grpc/watch_map_test.cc +++ b/test/extensions/config_subscription/grpc/watch_map_test.cc @@ -133,7 +133,7 @@ TEST(WatchMapTest, Basic) { TestUtility::TestOpaqueResourceDecoderImpl resource_decoder("cluster_name"); NiceMock config_validators; - WatchMap watch_map(false, "ClusterLoadAssignmentType", config_validators, {}); + WatchMap watch_map(false, "ClusterLoadAssignmentType", &config_validators, {}); Watch* watch = watch_map.addWatch(callbacks, resource_decoder); { @@ -207,7 +207,7 @@ TEST(WatchMapTest, Overlap) { TestUtility::TestOpaqueResourceDecoderImpl resource_decoder("cluster_name"); NiceMock config_validators; - WatchMap watch_map(false, "ClusterLoadAssignmentType", config_validators, {}); + WatchMap watch_map(false, "ClusterLoadAssignmentType", &config_validators, {}); Watch* watch1 = watch_map.addWatch(callbacks1, resource_decoder); Watch* watch2 = watch_map.addWatch(callbacks2, resource_decoder); @@ -276,7 +276,7 @@ TEST(WatchMapTest, CacheResourceAddResource) { NiceMock eds_resources_cache; const std::string eds_type_url = Config::getTypeUrl(); - WatchMap watch_map(false, eds_type_url, config_validators, + WatchMap watch_map(false, eds_type_url, &config_validators, makeOptRef(eds_resources_cache)); // The test uses 2 watchers to ensure that interest is kept regardless of // which watcher was the first to add a watch for the assignment. @@ -357,7 +357,7 @@ TEST(WatchMapTest, CacheResourceAddResource) { // WatchMap defers deletes and doesn't crash. class SameWatchRemoval : public testing::Test { public: - SameWatchRemoval() : watch_map_(false, "ClusterLoadAssignmentType", config_validators, {}) {} + SameWatchRemoval() : watch_map_(false, "ClusterLoadAssignmentType", &config_validators, {}) {} void SetUp() override { envoy::config::endpoint::v3::ClusterLoadAssignment alice; @@ -437,7 +437,7 @@ TEST(WatchMapTest, AddRemoveAdd) { TestUtility::TestOpaqueResourceDecoderImpl resource_decoder("cluster_name"); NiceMock config_validators; - WatchMap watch_map(false, "ClusterLoadAssignmentType", config_validators, {}); + WatchMap watch_map(false, "ClusterLoadAssignmentType", &config_validators, {}); Watch* watch1 = watch_map.addWatch(callbacks1, resource_decoder); Watch* watch2 = watch_map.addWatch(callbacks2, resource_decoder); @@ -494,7 +494,7 @@ TEST(WatchMapTest, UninterestingUpdate) { TestUtility::TestOpaqueResourceDecoderImpl resource_decoder("cluster_name"); NiceMock config_validators; - WatchMap watch_map(false, "ClusterLoadAssignmentType", config_validators, {}); + WatchMap watch_map(false, "ClusterLoadAssignmentType", &config_validators, {}); Watch* watch = watch_map.addWatch(callbacks, resource_decoder); watch_map.updateWatchInterest(watch, {"alice"}); @@ -539,7 +539,7 @@ TEST(WatchMapTest, WatchingEverything) { TestUtility::TestOpaqueResourceDecoderImpl resource_decoder("cluster_name"); NiceMock config_validators; - WatchMap watch_map(false, "ClusterLoadAssignmentType", config_validators, {}); + WatchMap watch_map(false, "ClusterLoadAssignmentType", &config_validators, {}); /*Watch* watch1 = */ watch_map.addWatch(callbacks1, resource_decoder); Watch* watch2 = watch_map.addWatch(callbacks2, resource_decoder); // watch1 never specifies any names, and so is treated as interested in everything. @@ -576,7 +576,7 @@ TEST(WatchMapTest, DeltaOnConfigUpdate) { TestUtility::TestOpaqueResourceDecoderImpl resource_decoder("cluster_name"); NiceMock config_validators; - WatchMap watch_map(false, "ClusterLoadAssignmentType", config_validators, {}); + WatchMap watch_map(false, "ClusterLoadAssignmentType", &config_validators, {}); Watch* watch1 = watch_map.addWatch(callbacks1, resource_decoder); Watch* watch2 = watch_map.addWatch(callbacks2, resource_decoder); Watch* watch3 = watch_map.addWatch(callbacks3, resource_decoder); @@ -610,7 +610,7 @@ TEST(WatchMapTest, DeltaOnConfigUpdate) { TEST(WatchMapTest, OnConfigUpdateFailed) { NiceMock config_validators; - WatchMap watch_map(false, "ClusterLoadAssignmentType", config_validators, {}); + WatchMap watch_map(false, "ClusterLoadAssignmentType", &config_validators, {}); // calling on empty map doesn't break watch_map.onConfigUpdateFailed(ConfigUpdateFailureReason::UpdateRejected, nullptr); @@ -632,7 +632,7 @@ TEST(WatchMapTest, OnConfigUpdateXdsTpGlobCollections) { TestUtility::TestOpaqueResourceDecoderImpl resource_decoder("cluster_name"); NiceMock config_validators; - WatchMap watch_map(false, "ClusterLoadAssignmentType", config_validators, {}); + WatchMap watch_map(false, "ClusterLoadAssignmentType", &config_validators, {}); Watch* watch = watch_map.addWatch(callbacks, resource_decoder); watch_map.updateWatchInterest(watch, {"xdstp://foo/bar/baz/*?some=thing&thing=some"}); @@ -677,7 +677,7 @@ TEST(WatchMapTest, OnConfigUpdateXdsTpSingletons) { TestUtility::TestOpaqueResourceDecoderImpl resource_decoder("cluster_name"); NiceMock config_validators; - WatchMap watch_map(false, "ClusterLoadAssignmentType", config_validators, {}); + WatchMap watch_map(false, "ClusterLoadAssignmentType", &config_validators, {}); Watch* watch = watch_map.addWatch(callbacks, resource_decoder); watch_map.updateWatchInterest(watch, {"xdstp://foo/bar/baz?some=thing&thing=some"}); @@ -718,7 +718,7 @@ TEST(WatchMapTest, OnConfigUpdateUsingNamespaces) { TestUtility::TestOpaqueResourceDecoderImpl resource_decoder("cluster_name"); NiceMock config_validators; - WatchMap watch_map(true, "ClusterLoadAssignmentType", config_validators, {}); + WatchMap watch_map(true, "ClusterLoadAssignmentType", &config_validators, {}); Watch* watch1 = watch_map.addWatch(callbacks1, resource_decoder); Watch* watch2 = watch_map.addWatch(callbacks2, resource_decoder); Watch* watch3 = watch_map.addWatch(callbacks3, resource_decoder); diff --git a/test/extensions/config_subscription/grpc/xds_failover_integration_test.cc b/test/extensions/config_subscription/grpc/xds_failover_integration_test.cc index de4948c91336..a1efe8febde9 100644 --- a/test/extensions/config_subscription/grpc/xds_failover_integration_test.cc +++ b/test/extensions/config_subscription/grpc/xds_failover_integration_test.cc @@ -906,4 +906,149 @@ TEST_P(XdsFailoverAdsIntegrationTest, ASSERT_TRUE(failover_xds_connection_->waitForDisconnect()); } } + +// Validation that when envoy.reloadable_features.xds_failover_to_primary_enabled is disabled +// and after the failover responds and then disconnected, Envoy will only +// try to reconnect to the failover. +// This test will be removed once envoy.reloadable_features.xds_failover_to_primary_enabled +// is deprecated. +TEST_P(XdsFailoverAdsIntegrationTest, NoPrimaryUseAfterFailoverResponse) { + // These tests are not executed with GoogleGrpc because they are flaky due to + // the large timeout values for retries. + SKIP_IF_GRPC_CLIENT(Grpc::ClientType::GoogleGrpc); +#ifdef ENVOY_ENABLE_UHV + // With UHV the finishGrpcStream() isn't detected as invalid frame because of + // no ":status" header, unless "envoy.reloadable_features.enable_universal_header_validator" + // is also enabled. + config_helper_.addRuntimeOverride("envoy.reloadable_features.enable_universal_header_validator", + "true"); +#endif + config_helper_.addRuntimeOverride("envoy.reloadable_features.xds_failover_to_primary_enabled", + "false"); + // Set a long LDS initial_fetch_timeout to prevent test flakiness when + // reconnecting to the failover multiple times. + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* lds_config = bootstrap.mutable_dynamic_resources()->mutable_lds_config(); + lds_config->mutable_initial_fetch_timeout()->set_seconds(100); + }); + initialize(); + + // 2 consecutive primary failures. + // Expect a connection to the primary. Reject the connection immediately. + primaryConnectionFailure(); + ASSERT_TRUE(xds_connection_->waitForDisconnect()); + // The CDS request fails when the primary disconnects. After that fetch the config + // dump to ensure that the retry timer kicks in. + // Expect another connection attempt to the primary. Reject the stream (gRPC failure) immediately. + // As this is a 2nd consecutive failure, it will trigger failover. + waitForPrimaryXdsRetryTimer(); + primaryConnectionFailure(); + ASSERT_TRUE(xds_connection_->waitForDisconnect()); + + // The CDS request fails when the primary disconnects. + test_server_->waitForCounterGe("cluster_manager.cds.update_failure", 2); + + AssertionResult result = + failover_xds_upstream_->waitForHttpConnection(*dispatcher_, failover_xds_connection_); + RELEASE_ASSERT(result, result.message()); + // Failover is healthy, start the ADS gRPC stream. + result = failover_xds_connection_->waitForNewStream(*dispatcher_, failover_xds_stream_); + RELEASE_ASSERT(result, result.message()); + failover_xds_stream_->startGrpcStream(); + + // Ensure basic flow with failover works. + EXPECT_TRUE(compareDiscoveryRequest(CdsTypeUrl, "", {}, {}, {}, true, + Grpc::Status::WellKnownGrpcStatus::Ok, "", + failover_xds_stream_.get())); + sendDiscoveryResponse( + CdsTypeUrl, {ConfigHelper::buildCluster("failover_cluster_0")}, + {ConfigHelper::buildCluster("failover_cluster_0")}, {}, "failover1", {}, + failover_xds_stream_.get()); + // Wait for an EDS request, and send its response. + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); + test_server_->waitForGaugeEq("cluster.failover_cluster_0.warming_state", 1); + // Ensure basic flow with failover works. + EXPECT_TRUE(compareDiscoveryRequest( + EdsTypeUrl, "", {"failover_cluster_0"}, {"failover_cluster_0"}, {}, false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", failover_xds_stream_.get())); + sendDiscoveryResponse( + EdsTypeUrl, {buildClusterLoadAssignment("failover_cluster_0")}, + {buildClusterLoadAssignment("failover_cluster_0")}, {}, "failover1", {}, + failover_xds_stream_.get()); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); + test_server_->waitForGaugeGe("cluster_manager.active_clusters", 2); + test_server_->waitForGaugeEq("cluster.failover_cluster_0.warming_state", 0); + EXPECT_EQ(2, test_server_->gauge("control_plane.connected_state")->value()); + EXPECT_TRUE(compareDiscoveryRequest(CdsTypeUrl, "failover1", {}, {}, {}, false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", + failover_xds_stream_.get())); + EXPECT_TRUE(compareDiscoveryRequest(LdsTypeUrl, "", {}, {}, {}, false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", + failover_xds_stream_.get())); + + // Envoy has received CDS and EDS responses, it means the failover is available. + // Now disconnect the failover source, this should result in an LDS failure. + // After that add a notification to the main thread to ensure that the retry timer kicks in. + failover_xds_stream_->finishGrpcStream(Grpc::Status::Internal); + test_server_->waitForCounterGe("listener_manager.lds.update_failure", 1); + absl::Notification notification; + test_server_->server().dispatcher().post([&]() { notification.Notify(); }); + notification.WaitForNotification(); + timeSystem().advanceTimeWait(std::chrono::milliseconds(1000)); + + // In this case (received a response), both EnvoyGrpc and GoogleGrpc keep the connection open. + result = failover_xds_connection_->waitForNewStream(*dispatcher_, failover_xds_stream_); + RELEASE_ASSERT(result, result.message()); + // Immediately fail the connection. + failover_xds_stream_->finishGrpcStream(Grpc::Status::Internal); + + // Ensure that Envoy still attempts to connect to the failover, + // and keep disconnecting a few times and validate that the primary + // connection isn't attempted. + for (int i = 1; i < 5; ++i) { + ASSERT_TRUE(failover_xds_connection_->waitForDisconnect()); + // Wait longer due to the fixed 5 seconds failover . + waitForPrimaryXdsRetryTimer(i, 6); + // EnvoyGrpc will disconnect if the gRPC stream is immediately closed (as + // done above). + result = failover_xds_upstream_->waitForHttpConnection(*dispatcher_, failover_xds_connection_); + RELEASE_ASSERT(result, result.message()); + result = failover_xds_connection_->waitForNewStream(*dispatcher_, failover_xds_stream_); + RELEASE_ASSERT(result, result.message()); + // Immediately fail the connection. + failover_xds_stream_->finishGrpcStream(Grpc::Status::Internal); + } + + // When EnvoyGrpc is used, no new connection to the primary will be attempted. + EXPECT_FALSE( + xds_upstream_->waitForHttpConnection(*dispatcher_, xds_connection_, std::chrono::seconds(1))); + + ASSERT_TRUE(failover_xds_connection_->waitForDisconnect()); + // Wait longer due to the fixed 5 seconds failover . + waitForPrimaryXdsRetryTimer(5, 6); + + // Allow a connection to the failover. + // Expect a connection to the failover when using EnvoyGrpc. + // In case GoogleGrpc is used the current connection will be reused (new stream). + result = failover_xds_upstream_->waitForHttpConnection(*dispatcher_, failover_xds_connection_); + RELEASE_ASSERT(result, result.message()); + result = failover_xds_connection_->waitForNewStream(*dispatcher_, failover_xds_stream_); + failover_xds_stream_->startGrpcStream(); + + // Validate that the initial requests with known versions are sent to the + // failover source. + const absl::flat_hash_map cds_eds_initial_resource_versions_map{ + {"failover_cluster_0", "failover1"}}; + const absl::flat_hash_map empty_initial_resource_versions_map; + EXPECT_TRUE(compareDiscoveryRequest( + CdsTypeUrl, "1", {}, {}, {}, true, Grpc::Status::WellKnownGrpcStatus::Ok, "", + failover_xds_stream_.get(), OptRef(cds_eds_initial_resource_versions_map))); + EXPECT_TRUE(compareDiscoveryRequest( + EdsTypeUrl, "1", {"failover_cluster_0"}, {"failover_cluster_0"}, {}, false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", failover_xds_stream_.get(), + OptRef(cds_eds_initial_resource_versions_map))); + EXPECT_TRUE(compareDiscoveryRequest( + LdsTypeUrl, "", {}, {}, {}, false, Grpc::Status::WellKnownGrpcStatus::Ok, "", + failover_xds_stream_.get(), OptRef(empty_initial_resource_versions_map))); +} } // namespace Envoy diff --git a/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc index 123ffd52a56d..f81b1da3ff94 100644 --- a/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc @@ -26,6 +26,7 @@ #include "test/test_common/logging.h" #include "test/test_common/resources.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/status_utility.h" #include "test/test_common/test_runtime.h" #include "test/test_common/test_time.h" #include "test/test_common/utility.h" @@ -94,7 +95,8 @@ class GrpcMuxImplTestBase : public testing::TestWithParam { const std::vector& resource_names, const std::string& version, bool first = false, const std::string& nonce = "", const Protobuf::int32 error_code = Grpc::Status::WellKnownGrpcStatus::Ok, - const std::string& error_message = "") { + const std::string& error_message = "", + Grpc::MockAsyncStream* async_stream = nullptr) { envoy::service::discovery::v3::DiscoveryRequest expected_request; if (first) { expected_request.mutable_node()->CopyFrom(local_info_.node()); @@ -113,7 +115,7 @@ class GrpcMuxImplTestBase : public testing::TestWithParam { error_detail->set_message(error_message); } EXPECT_CALL( - async_stream_, + async_stream ? *async_stream : async_stream_, sendMessageRaw_(Grpc::ProtoBufferEqIgnoreRepeatedFieldOrdering(expected_request), false)); } @@ -134,6 +136,9 @@ class GrpcMuxImplTestBase : public testing::TestWithParam { NiceMock random_; Grpc::MockAsyncClient* async_client_; Grpc::MockAsyncStream async_stream_; + // Used for tests invoking updateMuxSource(). + Grpc::MockAsyncClient* replaced_async_client_; + Grpc::MockAsyncStream replaced_async_stream_; NiceMock local_info_; CustomConfigValidatorsPtr config_validators_; std::unique_ptr grpc_mux_; @@ -1279,6 +1284,168 @@ TEST_P(GrpcMuxImplTest, AddRemoveSubscriptions) { } } +// Updating the mux object while being connected sends the correct requests. +TEST_P(GrpcMuxImplTest, MuxDynamicReplacementWhenConnected) { + replaced_async_client_ = new Grpc::MockAsyncClient(); + setup(); + InSequence s; + + auto foo_sub = makeWatch("type_url_foo", {"x", "y"}); + auto bar_sub = makeWatch("type_url_bar", {}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + expectSendMessage("type_url_bar", {}, ""); + grpc_mux_->start(); + EXPECT_EQ(1, control_plane_connected_state_.value()); + + // Switch the mux. + envoy::config::core::v3::ApiConfigSource empty_ads_config; + // Expect a disconnect from the original async_client and stream. + EXPECT_CALL(async_stream_, resetStream()); + // Expect establishing connection to the new client and stream. + EXPECT_CALL(*replaced_async_client_, startRaw(_, _, _, _)) + .WillOnce(Return(&replaced_async_stream_)); + // Expect the initial messages to be sent to the new stream. + expectSendMessage("type_url_foo", {"x", "y"}, "", true, "", Grpc::Status::WellKnownGrpcStatus::Ok, + "", &replaced_async_stream_); + expectSendMessage("type_url_bar", {}, "", false, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + &replaced_async_stream_); + EXPECT_OK(grpc_mux_->updateMuxSource( + /*primary_async_client=*/std::unique_ptr(replaced_async_client_), + /*failover_async_client=*/nullptr, + /*custom_config_validators=*/nullptr, + /*scope=*/*stats_.rootScope(), + /*backoff_strategy=*/ + std::make_unique( + SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_), + empty_ads_config)); + // Ending test, removing subscriptions for type_url_foo. + expectSendMessage("type_url_foo", {}, "", false, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + &replaced_async_stream_); +} + +// Updating the mux object after receiving a response, sends the correct requests. +TEST_P(GrpcMuxImplTest, MuxDynamicReplacementFetchingResources) { + replaced_async_client_ = new Grpc::MockAsyncClient(); + setup(); + InSequence s; + + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + auto foo_sub = makeWatch(type_url, {"x", "y"}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {"x", "y"}, "", true); + grpc_mux_->start(); + + // Send back a response for one of the resources. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([&load_assignment](const std::vector& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + const auto& expected_assignment = + dynamic_cast( + resources[0].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + return absl::OkStatus(); + })); + expectSendMessage(type_url, {"x", "y"}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + + // Switch the mux. + envoy::config::core::v3::ApiConfigSource empty_ads_config; + // Expect a disconnect from the original async_client and stream. + EXPECT_CALL(async_stream_, resetStream()); + // Expect establishing connection to the new client and stream. + EXPECT_CALL(*replaced_async_client_, startRaw(_, _, _, _)) + .WillOnce(Return(&replaced_async_stream_)); + // Expect the initial message to be sent to the new stream. + expectSendMessage(type_url, {"x", "y"}, "1", true, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + &replaced_async_stream_); + EXPECT_OK(grpc_mux_->updateMuxSource( + /*primary_async_client=*/std::unique_ptr(replaced_async_client_), + /*failover_async_client=*/nullptr, + /*custom_config_validators=*/std::make_unique>(), + /*scope=*/*stats_.rootScope(), + /*backoff_strategy=*/ + std::make_unique( + SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_), + empty_ads_config)); + + // Send a response to resource "y" on the replaced mux. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("2"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("y"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "2")) + .WillOnce(Invoke([&load_assignment](const std::vector& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + const auto& expected_assignment = + dynamic_cast( + resources[0].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + return absl::OkStatus(); + })); + expectSendMessage(type_url, {"x", "y"}, "2", false, "", Grpc::Status::WellKnownGrpcStatus::Ok, + "", &replaced_async_stream_); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + + // Ending test, removing subscriptions for the subscription. + expectSendMessage(type_url, {}, "2", false, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + &replaced_async_stream_); +} + +// Updating the mux object with wrong rate limit settings is rejected. +TEST_P(GrpcMuxImplTest, RejectMuxDynamicReplacementRateLimitSettingsError) { + replaced_async_client_ = new Grpc::MockAsyncClient(); + setup(); + InSequence s; + + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + auto foo_sub = makeWatch(type_url, {"x", "y"}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {"x", "y"}, "", true); + grpc_mux_->start(); + EXPECT_EQ(1, control_plane_connected_state_.value()); + + // Switch the mux. + envoy::config::core::v3::ApiConfigSource ads_config_wrong_settings; + envoy::config::core::v3::RateLimitSettings* rate_limits = + ads_config_wrong_settings.mutable_rate_limit_settings(); + rate_limits->mutable_max_tokens()->set_value(500); + rate_limits->mutable_fill_rate()->set_value(std::numeric_limits::quiet_NaN()); + // No disconnect and replacement of the original async_client. + EXPECT_CALL(async_stream_, resetStream()).Times(0); + EXPECT_CALL(*replaced_async_client_, startRaw(_, _, _, _)).Times(0); + EXPECT_FALSE(grpc_mux_ + ->updateMuxSource( + /*primary_async_client=*/std::unique_ptr( + replaced_async_client_), + /*failover_async_client=*/nullptr, + /*custom_config_validators=*/nullptr, + /*scope=*/*stats_.rootScope(), + /*backoff_strategy=*/ + std::make_unique( + SubscriptionFactory::RetryInitialDelayMs, + SubscriptionFactory::RetryMaxDelayMs, random_), + ads_config_wrong_settings) + .ok()); + // Ending test, removing subscriptions for type_url_foo. + expectSendMessage(type_url, {}, "", false, "", Grpc::Status::WellKnownGrpcStatus::Ok, "", + &async_stream_); +} + class NullGrpcMuxImplTest : public testing::Test { public: NullGrpcMuxImplTest() : null_mux_(std::make_unique()) {} diff --git a/test/extensions/filters/common/expr/context_test.cc b/test/extensions/filters/common/expr/context_test.cc index c766d47e258d..f7cec4fb53a2 100644 --- a/test/extensions/filters/common/expr/context_test.cc +++ b/test/extensions/filters/common/expr/context_test.cc @@ -777,6 +777,8 @@ TEST(Context, FilterStateAttributes) { StreamInfo::FilterStateImpl filter_state(StreamInfo::FilterState::LifeSpan::FilterChain); ProtobufWkt::Arena arena; FilterStateWrapper wrapper(arena, filter_state); + auto status_or = wrapper.ListKeys(&arena); + EXPECT_EQ(status_or.status().message(), "ListKeys() is not implemented"); const std::string key = "filter_state_key"; const std::string serialized = "filter_state_value"; @@ -941,6 +943,21 @@ TEST(Context, XDSAttributes) { } } +TEST(Context, EmptyXdsWrapper) { + Protobuf::Arena arena; + XDSWrapper wrapper(arena, nullptr, nullptr); + + { + const auto value = wrapper[CelValue::CreateStringView(Node)]; + EXPECT_FALSE(value.has_value()); + } + + { + const auto value = wrapper[CelValue::CreateStringView(ClusterName)]; + EXPECT_FALSE(value.has_value()); + } +} + } // namespace } // namespace Expr } // namespace Common diff --git a/test/extensions/filters/common/ratelimit_config/BUILD b/test/extensions/filters/common/ratelimit_config/BUILD new file mode 100644 index 000000000000..bffabf03a6cd --- /dev/null +++ b/test/extensions/filters/common/ratelimit_config/BUILD @@ -0,0 +1,37 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", + "envoy_proto_library", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_proto_library( + name = "ratelimit_config_test_proto", + srcs = ["ratelimit_config_test.proto"], + deps = [ + "@envoy_api//envoy/config/route/v3:pkg", + ], +) + +envoy_cc_test( + name = "ratelimit_config_test", + srcs = ["ratelimit_config_test.cc"], + deps = [ + ":ratelimit_config_test_proto_cc_proto", + "//source/common/http:header_map_lib", + "//source/common/protobuf:utility_lib", + "//source/common/router:config_lib", + "//source/extensions/filters/common/ratelimit_config:ratelimit_config_lib", + "//test/mocks/http:http_mocks", + "//test/mocks/ratelimit:ratelimit_mocks", + "//test/mocks/router:router_mocks", + "//test/mocks/server:instance_mocks", + "//test/test_common:registry_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.cc b/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.cc new file mode 100644 index 000000000000..aea1832038bc --- /dev/null +++ b/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.cc @@ -0,0 +1,1121 @@ +#include +#include +#include + +#include "envoy/config/route/v3/route.pb.h" +#include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/config/route/v3/route_components.pb.validate.h" + +#include "source/common/http/header_map_impl.h" +#include "source/common/network/address_impl.h" +#include "source/common/protobuf/utility.h" +#include "source/extensions/filters/common/ratelimit_config/ratelimit_config.h" + +#include "test/extensions/filters/common/ratelimit_config/ratelimit_config_test.pb.h" +#include "test/extensions/filters/common/ratelimit_config/ratelimit_config_test.pb.validate.h" +#include "test/mocks/http/mocks.h" +#include "test/mocks/ratelimit/mocks.h" +#include "test/mocks/router/mocks.h" +#include "test/mocks/server/instance.h" +#include "test/test_common/printers.h" +#include "test/test_common/registry.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace RateLimit { +namespace { + +ProtoRateLimit parseRateLimitFromV3Yaml(const std::string& yaml_string) { + ProtoRateLimit rate_limit; + TestUtility::loadFromYaml(yaml_string, rate_limit); + TestUtility::validate(rate_limit); + return rate_limit; +} + +TEST(BadRateLimitConfiguration, MissingActions) { + EXPECT_THROW_WITH_REGEX(parseRateLimitFromV3Yaml("{}"), EnvoyException, + "value must contain at least"); +} + +TEST(BadRateLimitConfiguration, ActionsMissingRequiredFields) { + const std::string yaml_one = R"EOF( +actions: +- request_headers: {} + )EOF"; + + EXPECT_THROW_WITH_REGEX(parseRateLimitFromV3Yaml(yaml_one), EnvoyException, + "value length must be at least"); + + const std::string yaml_two = R"EOF( +actions: +- request_headers: + header_name: test + )EOF"; + + EXPECT_THROW_WITH_REGEX(parseRateLimitFromV3Yaml(yaml_two), EnvoyException, + "value length must be at least"); + + const std::string yaml_three = R"EOF( +actions: +- request_headers: + descriptor_key: test + )EOF"; + + EXPECT_THROW_WITH_REGEX(parseRateLimitFromV3Yaml(yaml_three), EnvoyException, + "value length must be at least"); +} + +class RateLimitConfigTest : public testing::Test { +public: + void setupTest(const std::string& yaml) { + test::extensions::filters::common::ratelimit_config::TestRateLimitConfig proto_config; + TestUtility::loadFromYaml(yaml, proto_config); + config_ = std::make_unique( + proto_config.rate_limits(), factory_context_, creation_status_); + stream_info_.downstream_connection_info_provider_->setRemoteAddress(default_remote_address_); + ON_CALL(Const(stream_info_), route()).WillByDefault(testing::Return(route_)); + } + + NiceMock factory_context_; + ProtobufMessage::NullValidationVisitorImpl any_validation_visitor_; + absl::Status creation_status_{}; + std::unique_ptr config_; + Http::TestRequestHeaderMapImpl headers_; + std::shared_ptr route_{new NiceMock()}; + Network::Address::InstanceConstSharedPtr default_remote_address_{ + new Network::Address::Ipv4Instance("10.0.0.1")}; + NiceMock stream_info_; +}; + +TEST_F(RateLimitConfigTest, DisableKeyIsNotAllowed) { + { + const std::string yaml = R"EOF( + rate_limits: + - actions: + - remote_address: {} + stage: 2 + disable_key: anything + limit: + dynamic_metadata: + metadata_key: + key: key + path: + - key: key + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); + setupTest(yaml); + EXPECT_FALSE(creation_status_.ok()); + EXPECT_EQ(creation_status_.message(), + "'stage' field and 'disable_key' field are not supported"); + } +} + +TEST_F(RateLimitConfigTest, LimitIsNotAllowed) { + { + const std::string yaml = R"EOF( + rate_limits: + - actions: + - remote_address: {} + limit: + dynamic_metadata: + metadata_key: + key: key + path: + - key: key + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); + setupTest(yaml); + EXPECT_FALSE(creation_status_.ok()); + EXPECT_EQ(creation_status_.message(), "'limit' field is not supported"); + } +} + +TEST_F(RateLimitConfigTest, NoAction) { + { + const std::string yaml = R"EOF( +actions: +- {} + )EOF"; + + ProtoRateLimit rate_limit; + TestUtility::loadFromYaml(yaml, rate_limit); + + absl::Status creation_status; + RateLimitPolicy policy(rate_limit, factory_context_, creation_status); + + EXPECT_TRUE(absl::StartsWith(creation_status.message(), "Unsupported rate limit action:")); + } + + { + const std::string yaml = R"EOF( + rate_limits: + - actions: + - remote_address: {} + - {} + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); + setupTest(yaml); + + EXPECT_TRUE(absl::StartsWith(creation_status_.message(), "Unsupported rate limit action:")); + } +} + +TEST_F(RateLimitConfigTest, EmptyRateLimit) { + const std::string yaml = R"EOF( +rate_limits: [] + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); + setupTest(yaml); + + EXPECT_TRUE(config_->empty()); +} + +TEST_F(RateLimitConfigTest, SinglePolicy) { + const std::string yaml = R"EOF( + rate_limits: + - actions: + - remote_address: {} + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); + setupTest(yaml); + + EXPECT_EQ(1U, config_->size()); + + std::vector descriptors; + config_->populateDescriptors(headers_, stream_info_, "", descriptors); + EXPECT_THAT(std::vector({{{{"remote_address", "10.0.0.1"}}}}), + testing::ContainerEq(descriptors)); +} + +TEST_F(RateLimitConfigTest, MultiplePoliciesAndMultipleActions) { + const std::string yaml = R"EOF( + rate_limits: + - actions: + - remote_address: {} + - destination_cluster: {} + - actions: + - destination_cluster: {} + )EOF"; + + setupTest(yaml); + + std::vector descriptors; + + config_->populateDescriptors(headers_, stream_info_, "", descriptors); + + EXPECT_THAT(std::vector( + {Envoy::RateLimit::LocalDescriptor{ + {{"remote_address", "10.0.0.1"}, {"destination_cluster", "fake_cluster"}}}, + Envoy::RateLimit::LocalDescriptor{{{"destination_cluster", "fake_cluster"}}}}), + testing::ContainerEq(descriptors)); +} + +class RateLimitPolicyTest : public testing::Test { +public: + void setupTest(const std::string& yaml) { + rate_limit_entry_ = std::make_unique(parseRateLimitFromV3Yaml(yaml), + factory_context_, creation_status_); + descriptors_.clear(); + stream_info_.downstream_connection_info_provider_->setRemoteAddress(default_remote_address_); + ON_CALL(Const(stream_info_), route()).WillByDefault(testing::Return(route_)); + } + + NiceMock factory_context_; + std::unique_ptr rate_limit_entry_; + absl::Status creation_status_{}; + Http::TestRequestHeaderMapImpl headers_; + std::shared_ptr route_{new NiceMock()}; + + std::vector descriptors_; + Network::Address::InstanceConstSharedPtr default_remote_address_{ + new Network::Address::Ipv4Instance("10.0.0.1")}; + NiceMock stream_info_; +}; + +class RateLimitPolicyIpv6Test : public testing::Test { +public: + void setupTest(const std::string& yaml) { + absl::Status creation_status; + rate_limit_entry_ = std::make_unique(parseRateLimitFromV3Yaml(yaml), + factory_context_, creation_status); + THROW_IF_NOT_OK(creation_status); // NOLINT + descriptors_.clear(); + stream_info_.downstream_connection_info_provider_->setRemoteAddress(default_remote_address_); + ON_CALL(Const(stream_info_), route()).WillByDefault(testing::Return(route_)); + } + + NiceMock factory_context_; + std::unique_ptr rate_limit_entry_; + Http::TestRequestHeaderMapImpl headers_; + std::vector descriptors_; + std::shared_ptr route_{new NiceMock()}; + + Network::Address::InstanceConstSharedPtr default_remote_address_{ + new Network::Address::Ipv6Instance("2001:abcd:ef01:2345:6789:abcd:ef01:234")}; + NiceMock stream_info_; +}; + +TEST_F(RateLimitPolicyTest, RemoteAddress) { + const std::string yaml = R"EOF( +actions: +- remote_address: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"remote_address", "10.0.0.1"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MaskedRemoteAddressIpv4Default) { + const std::string yaml = R"EOF( +actions: +- masked_remote_address: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector( + {{{{"masked_remote_address", "10.0.0.1/32"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MaskedRemoteAddressIpv4) { + const std::string yaml = R"EOF( +actions: +- masked_remote_address: + v4_prefix_mask_len: 16 + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector( + {{{{"masked_remote_address", "10.0.0.0/16"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyIpv6Test, MaskedRemoteAddressIpv6Default) { + const std::string yaml = R"EOF( +actions: +- masked_remote_address: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector( + {{{{"masked_remote_address", "2001:abcd:ef01:2345:6789:abcd:ef01:234/128"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyIpv6Test, MaskedRemoteAddressIpv6) { + const std::string yaml = R"EOF( +actions: +- masked_remote_address: + v6_prefix_mask_len: 64 + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector( + {{{{"masked_remote_address", "2001:abcd:ef01:2345::/64"}}}}), + testing::ContainerEq(descriptors_)); +} + +// Verify no descriptor is emitted if remote is a pipe. +TEST_F(RateLimitPolicyTest, PipeAddress) { + const std::string yaml = R"EOF( +actions: +- remote_address: {} + )EOF"; + + setupTest(yaml); + + stream_info_.downstream_connection_info_provider_->setRemoteAddress( + *Network::Address::PipeInstance::create("/hello")); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, SourceService) { + const std::string yaml = R"EOF( +actions: +- source_cluster: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector({{{{"source_cluster", "service_cluster"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, DestinationService) { + const std::string yaml = R"EOF( +actions: +- destination_cluster: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector({{{{"destination_cluster", "fake_cluster"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, RequestHeaders) { + const std::string yaml = R"EOF( +actions: +- request_headers: + header_name: x-header-name + descriptor_key: my_header_name + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector({{{{"my_header_name", "test_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +// Validate that a descriptor is added if the missing request header +// has skip_if_absent set to true +TEST_F(RateLimitPolicyTest, RequestHeadersWithSkipIfAbsent) { + const std::string yaml = R"EOF( +actions: +- request_headers: + header_name: x-header-name + descriptor_key: my_header_name + skip_if_absent: false +- request_headers: + header_name: x-header + descriptor_key: my_header + skip_if_absent: true + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector({{{{"my_header_name", "test_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +// Tests if the descriptors are added if one of the headers is missing +// and skip_if_absent is set to default value which is false +TEST_F(RateLimitPolicyTest, RequestHeadersWithDefaultSkipIfAbsent) { + const std::string yaml = R"EOF( +actions: +- request_headers: + header_name: x-header-name + descriptor_key: my_header_name + skip_if_absent: false +- request_headers: + header_name: x-header + descriptor_key: my_header + skip_if_absent: false + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{"x-header-test", "test_value"}}; + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, RequestHeadersNoMatch) { + const std::string yaml = R"EOF( +actions: +- request_headers: + header_name: x-header + descriptor_key: my_header_name + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, RateLimitKey) { + const std::string yaml = R"EOF( +actions: +- generic_key: + descriptor_value: fake_key + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"generic_key", "fake_key"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, GenericKeyWithSetDescriptorKey) { + const std::string yaml = R"EOF( +actions: +- generic_key: + descriptor_key: fake_key + descriptor_value: fake_value + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, GenericKeyWithEmptyDescriptorKey) { + const std::string yaml = R"EOF( +actions: +- generic_key: + descriptor_key: "" + descriptor_value: fake_value + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"generic_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MetaDataMatchDynamicSourceByDefault) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + default_value: fake_value + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + test: + prop: foo + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "foo"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MetaDataMatchDynamicSource) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + default_value: fake_value + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + source: DYNAMIC + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + test: + prop: foo + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "foo"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MetaDataMatchRouteEntrySource) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + default_value: fake_value + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + source: ROUTE_ENTRY + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + test: + prop: foo + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, route_->metadata_); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "foo"}}}}), + testing::ContainerEq(descriptors_)); +} + +// Tests that the default_value is used in the descriptor when the metadata_key is empty. +TEST_F(RateLimitPolicyTest, MetaDataNoMatchWithDefaultValue) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + default_value: fake_value + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + another_key: + prop: foo + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MetaDataNoMatch) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + another_key: + prop: foo + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, MetaDataEmptyValue) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + test: + prop: "" + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +// Tests that no descriptors are generated when both the metadata_key and default_value are empty. +TEST_F(RateLimitPolicyTest, MetaDataAndDefaultValueEmpty) { + const std::string yaml = R"EOF( +actions: +- generic_key: + descriptor_key: fake_key + descriptor_value: fake_value +- metadata: + descriptor_key: fake_key + default_value: "" + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + another_key: + prop: "" + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +// Tests that no descriptor is generated when both the metadata_key and default_value are empty, +// and skip_if_absent is set to true. +TEST_F(RateLimitPolicyTest, MetaDataAndDefaultValueEmptySkipIfAbsent) { + const std::string yaml = R"EOF( +actions: +- generic_key: + descriptor_key: fake_key + descriptor_value: fake_value +- metadata: + descriptor_key: fake_key + default_value: "" + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + skip_if_absent: true + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + another_key: + prop: "" + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, MetaDataNonStringNoMatch) { + const std::string yaml = R"EOF( +actions: +- metadata: + descriptor_key: fake_key + metadata_key: + key: 'envoy.xxx' + path: + - key: test + - key: prop + )EOF"; + + setupTest(yaml); + + std::string metadata_yaml = R"EOF( +filter_metadata: + envoy.xxx: + test: + prop: + foo: bar + )EOF"; + + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, HeaderValueMatch) { + const std::string yaml = R"EOF( +actions: +- header_value_match: + descriptor_value: fake_value + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"header_match", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, HeaderValueMatchDescriptorKey) { + const std::string yaml = R"EOF( +actions: +- header_value_match: + descriptor_key: fake_key + descriptor_value: fake_value + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, HeaderValueMatchNoMatch) { + const std::string yaml = R"EOF( +actions: +- header_value_match: + descriptor_value: fake_value + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "not_same_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, HeaderValueMatchHeadersNotPresent) { + const std::string yaml = R"EOF( +actions: +- header_value_match: + descriptor_value: fake_value + expect_match: false + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "not_same_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"header_match", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, HeaderValueMatchHeadersPresent) { + const std::string yaml = R"EOF( +actions: +- header_value_match: + descriptor_value: fake_value + expect_match: false + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, QueryParameterValueMatch) { + const std::string yaml = R"EOF( +actions: +- query_parameter_value_match: + descriptor_value: fake_value + query_parameters: + - name: x-parameter-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":path", "/?x-parameter-name=test_value"}}; + + rate_limit_entry_->populateDescriptors(header, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"query_match", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, QueryParameterValueMatchDescriptorKey) { + const std::string yaml = R"EOF( +actions: +- query_parameter_value_match: + descriptor_key: fake_key + descriptor_value: fake_value + query_parameters: + - name: x-parameter-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":path", "/?x-parameter-name=test_value"}}; + + rate_limit_entry_->populateDescriptors(header, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, QueryParameterValueMatchNoMatch) { + const std::string yaml = R"EOF( +actions: +- query_parameter_value_match: + descriptor_value: fake_value + query_parameters: + - name: x-parameter-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":path", "/?x-parameter-name=not_same_value"}}; + + rate_limit_entry_->populateDescriptors(header, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, QueryParameterValueMatchExpectNoMatch) { + const std::string yaml = R"EOF( +actions: +- query_parameter_value_match: + descriptor_value: fake_value + expect_match: false + query_parameters: + - name: x-parameter-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":path", "/?x-parameter-name=not_same_value"}}; + + rate_limit_entry_->populateDescriptors(header, stream_info_, "", descriptors_); + + EXPECT_THAT(std::vector({{{{"query_match", "fake_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, QueryParameterValueMatchExpectNoMatchFailed) { + const std::string yaml = R"EOF( +actions: +- query_parameter_value_match: + descriptor_value: fake_value + expect_match: false + query_parameters: + - name: x-parameter-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":path", "/?x-parameter-name=test_value"}}; + + rate_limit_entry_->populateDescriptors(header, stream_info_, "", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, CompoundActions) { + const std::string yaml = R"EOF( +actions: +- destination_cluster: {} +- source_cluster: {} + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector( + {{{{"destination_cluster", "fake_cluster"}, {"source_cluster", "service_cluster"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, CompoundActionsNoDescriptor) { + const std::string yaml = R"EOF( +actions: +- destination_cluster: {} +- header_value_match: + descriptor_value: fake_value + headers: + - name: x-header-name + string_match: + exact: test_value + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +const std::string RequestHeaderMatchInputDescriptor = R"EOF( +actions: +- extension: + name: my_header_name + typed_config: + "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput + header_name: x-header-name + )EOF"; + +TEST_F(RateLimitPolicyTest, RequestMatchInput) { + setupTest(RequestHeaderMatchInputDescriptor); + headers_.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_THAT( + std::vector({{{{"my_header_name", "test_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyTest, RequestMatchInputEmpty) { + setupTest(RequestHeaderMatchInputDescriptor); + headers_.setCopy(Http::LowerCaseString("x-header-name"), ""); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_FALSE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyTest, RequestMatchInputSkip) { + setupTest(RequestHeaderMatchInputDescriptor); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + + EXPECT_TRUE(descriptors_.empty()); +} + +class ExtensionDescriptorFactory : public Envoy::RateLimit::DescriptorProducerFactory { +public: + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + std::string name() const override { return "test.descriptor_producer"; } + + Envoy::RateLimit::DescriptorProducerPtr + createDescriptorProducerFromProto(const Protobuf::Message&, + Server::Configuration::CommonFactoryContext&) override { + return return_valid_producer_ ? std::make_unique() : nullptr; + } + bool return_valid_producer_{true}; +}; + +TEST_F(RateLimitPolicyTest, ExtensionDescriptorProducer) { + const std::string ExtensionDescriptor = R"EOF( +actions: +- extension: + name: test.descriptor_producer + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + key: value + )EOF"; + + { + ExtensionDescriptorFactory factory; + Registry::InjectFactory registration(factory); + + setupTest(ExtensionDescriptor); + + rate_limit_entry_->populateDescriptors(headers_, stream_info_, "service_cluster", descriptors_); + EXPECT_THAT( + std::vector({{{{"source_cluster", "service_cluster"}}}}), + testing::ContainerEq(descriptors_)); + } + + { + ExtensionDescriptorFactory factory; + factory.return_valid_producer_ = false; + Registry::InjectFactory registration(factory); + + setupTest(ExtensionDescriptor); + + EXPECT_TRUE( + absl::StartsWith(creation_status_.message(), "Rate limit descriptor extension failed:")); + } +} + +} // namespace +} // namespace RateLimit +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.proto b/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.proto new file mode 100644 index 000000000000..08d48af9cb86 --- /dev/null +++ b/test/extensions/filters/common/ratelimit_config/ratelimit_config_test.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package test.extensions.filters.common.ratelimit_config; + +import "envoy/config/route/v3/route_components.proto"; + +message TestRateLimitConfig { + repeated envoy.config.route.v3.RateLimit rate_limits = 1; +} diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index dc2be25f4fce..f872849157ef 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -40,7 +40,7 @@ class ProxyFilterIntegrationTest : public testing::TestWithParam inject_factory(factory); + Registry::InjectFactory::forceAllowDuplicates(); + config_helper_.addRuntimeOverride("envoy.enable_dfp_dns_trace", "true"); + useAccessLog("%RESPONSE_CODE_DETAILS%"); + + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); + + config_helper_.prependFilter(fmt::format(R"EOF( + name: stream-info-to-headers-filter +)EOF")); + + upstream_tls_ = false; // upstream creation doesn't handle autonomous_upstream_ + autonomous_upstream_ = true; + std::string resolver_config = R"EOF( + typed_dns_resolver_config: + name: envoy.network.dns_resolver.getaddrinfo + typed_config: + "@type": type.googleapis.com/envoy.extensions.network.dns_resolver.getaddrinfo.v3.GetAddrInfoDnsResolverConfig)EOF"; + initializeWithArgs(1024, 1024, "", resolver_config, false, 0.000000001); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("503", response->headers().getStatusValue()); + EXPECT_THAT(waitForAccessLog(access_log_name_), + HasSubstr("dns_resolution_failure{resolve_timeout:")); +} + +TEST_P(ProxyFilterIntegrationTest, GetAddrInfoResolveTimeoutWithoutTrace) { + Network::OverrideAddrInfoDnsResolverFactory factory; + Registry::InjectFactory inject_factory(factory); + Registry::InjectFactory::forceAllowDuplicates(); + useAccessLog("%RESPONSE_CODE_DETAILS%"); + + setDownstreamProtocol(Http::CodecType::HTTP2); + setUpstreamProtocol(Http::CodecType::HTTP2); + + config_helper_.prependFilter(fmt::format(R"EOF( + name: stream-info-to-headers-filter +)EOF")); + + upstream_tls_ = false; // upstream creation doesn't handle autonomous_upstream_ + autonomous_upstream_ = true; + std::string resolver_config = R"EOF( + typed_dns_resolver_config: + name: envoy.network.dns_resolver.getaddrinfo + typed_config: + "@type": type.googleapis.com/envoy.extensions.network.dns_resolver.getaddrinfo.v3.GetAddrInfoDnsResolverConfig)EOF"; + initializeWithArgs(1024, 1024, "", resolver_config, false, 0.000000001); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("503", response->headers().getStatusValue()); + EXPECT_THAT(waitForAccessLog(access_log_name_), + HasSubstr("dns_resolution_failure{resolve_timeout}")); +} + TEST_P(ProxyFilterIntegrationTest, ParallelRequests) { setDownstreamProtocol(Http::CodecType::HTTP2); setUpstreamProtocol(Http::CodecType::HTTP2); diff --git a/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.cc b/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.cc index 23ff9bd71207..88b73ee94460 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.cc @@ -4,7 +4,7 @@ namespace Envoy { namespace Network { absl::Mutex TestResolver::resolution_mutex_; -std::list dns_override)>> +std::list dns_override)>> TestResolver::blocked_resolutions_; REGISTER_FACTORY(TestResolverFactory, DnsResolverFactory); diff --git a/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h b/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h index 72487998872b..738c5bc6d954 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h +++ b/test/extensions/filters/http/dynamic_forward_proxy/test_resolver.h @@ -27,7 +27,7 @@ class TestResolver : public GetAddrInfoDnsResolver { if (blocked_resolutions_.empty()) { continue; } - auto run = blocked_resolutions_.front(); + auto run = std::move(blocked_resolutions_.front()); blocked_resolutions_.pop_front(); run(dns_override); return; @@ -36,21 +36,25 @@ class TestResolver : public GetAddrInfoDnsResolver { ActiveDnsQuery* resolve(const std::string& dns_name, DnsLookupFamily dns_lookup_family, ResolveCb callback) override { - auto new_query = new PendingQuery(dns_name, dns_lookup_family, callback, mutex_); - + std::unique_ptr new_query = + std::make_unique(dns_name, dns_lookup_family, callback); + PendingQuery* raw_new_query = new_query.get(); absl::MutexLock guard(&resolution_mutex_); - blocked_resolutions_.push_back([&, new_query](absl::optional dns_override) { - absl::MutexLock guard(&mutex_); - if (dns_override.has_value()) { - *const_cast(&new_query->dns_name_) = dns_override.value(); - } - pending_queries_.push_back({std::unique_ptr{new_query}, absl::nullopt}); - }); - return new_query; + blocked_resolutions_.push_back( + [&, query = std::move(new_query)](absl::optional dns_override) mutable { + absl::MutexLock guard(&mutex_); + if (dns_override.has_value()) { + *const_cast(&query->dns_name_) = dns_override.value(); + } + // Add a dummy trace for test coverage. + query->addTrace(100); + pending_queries_.push_back(PendingQueryInfo{std::move(query), absl::nullopt}); + }); + return raw_new_query; } static absl::Mutex resolution_mutex_; - static std::list dns_override)>> + static std::list dns_override)>> blocked_resolutions_ ABSL_GUARDED_BY(resolution_mutex_); }; diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc index e229fc88b850..7762d38d5b27 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc @@ -4334,6 +4334,43 @@ TEST_P(ExtProcIntegrationTest, ObservabilityModeWithFullResponse) { timeSystem().advanceTimeWaitImpl(std::chrono::milliseconds(deferred_close_timeout_ms)); } +TEST_P(ExtProcIntegrationTest, ObservabilityModeWithFullRequestAndTimeout) { + proto_config_.set_observability_mode(true); + uint32_t deferred_close_timeout_ms = 2000; + proto_config_.mutable_deferred_close_timeout()->set_seconds(deferred_close_timeout_ms / 1000); + + proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::STREAMED); + proto_config_.mutable_processing_mode()->set_request_trailer_mode(ProcessingMode::SEND); + proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); + + initializeConfig(); + HttpIntegrationTest::initialize(); + auto response = sendDownstreamRequestWithBodyAndTrailer("Hello"); + + processRequestHeadersMessage(*grpc_upstreams_[0], true, + [this](const HttpHeaders&, HeadersResponse&) { + // Advance 400 ms. Default timeout is 200ms + timeSystem().advanceTimeWaitImpl(400ms); + return false; + }); + processRequestBodyMessage(*grpc_upstreams_[0], false, [this](const HttpBody&, BodyResponse&) { + // Advance 400 ms. Default timeout is 200ms + timeSystem().advanceTimeWaitImpl(400ms); + return false; + }); + processRequestTrailersMessage(*grpc_upstreams_[0], false, + [this](const HttpTrailers&, TrailersResponse&) { + // Advance 400 ms. Default timeout is 200ms + timeSystem().advanceTimeWaitImpl(400ms); + return false; + }); + + handleUpstreamRequest(); + verifyDownstreamResponse(*response, 200); + + timeSystem().advanceTimeWaitImpl(std::chrono::milliseconds(deferred_close_timeout_ms - 1200)); +} + TEST_P(ExtProcIntegrationTest, ObservabilityModeWithLogging) { proto_config_.set_observability_mode(true); diff --git a/test/extensions/filters/http/ext_proc/filter_test.cc b/test/extensions/filters/http/ext_proc/filter_test.cc index 148d47e2de26..5193c769b5df 100644 --- a/test/extensions/filters/http/ext_proc/filter_test.cc +++ b/test/extensions/filters/http/ext_proc/filter_test.cc @@ -1068,8 +1068,6 @@ TEST_F(HttpFilterTest, PostAndChangeRequestBodyBufferedComesFast) { Buffer::OwnedImpl buffered_data; setUpDecodingBuffering(buffered_data); - // Buffering and callback isn't complete so we should watermark - EXPECT_CALL(decoder_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_1, false)); buffered_data.add(req_data_1); @@ -1082,7 +1080,6 @@ TEST_F(HttpFilterTest, PostAndChangeRequestBodyBufferedComesFast) { EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_4, true)); buffered_data.add(req_data_4); - EXPECT_CALL(decoder_callbacks_, onDecoderFilterBelowWriteBufferLowWatermark()); processRequestHeaders(true, absl::nullopt); processRequestBody([](const HttpBody& req_body, ProcessingResponse&, BodyResponse&) { @@ -1136,15 +1133,11 @@ TEST_F(HttpFilterTest, PostAndChangeRequestBodyBufferedComesALittleFast) { Buffer::OwnedImpl buffered_data; setUpDecodingBuffering(buffered_data); - // Buffering and callback isn't complete so we should watermark - EXPECT_CALL(decoder_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_1, false)); buffered_data.add(req_data_1); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_2, false)); buffered_data.add(req_data_2); - // Now the headers response comes in before we get all the data - EXPECT_CALL(decoder_callbacks_, onDecoderFilterBelowWriteBufferLowWatermark()); processRequestHeaders(true, absl::nullopt); EXPECT_EQ(FilterDataStatus::StopIterationAndBuffer, filter_->decodeData(req_data_3, false)); @@ -1454,15 +1447,11 @@ TEST_F(HttpFilterTest, PostFastRequestPartialBuffering) { Buffer::OwnedImpl buffered_data; setUpDecodingBuffering(buffered_data); - // Buffering and callback isn't complete so we should watermark - EXPECT_CALL(decoder_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_1, false)); buffered_data.add(req_data_1); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_2, true)); buffered_data.add(req_data_2); - // Now the headers response comes in and we are all done - EXPECT_CALL(decoder_callbacks_, onDecoderFilterBelowWriteBufferLowWatermark()); processRequestHeaders(true, absl::nullopt); processRequestBody([](const HttpBody& req_body, ProcessingResponse&, BodyResponse&) { @@ -1518,8 +1507,6 @@ TEST_F(HttpFilterTest, PostFastAndBigRequestPartialBuffering) { expected_request_data.add(req_data_1); expected_request_data.add(req_data_2); - // Buffering and callback isn't complete so we should watermark - EXPECT_CALL(decoder_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_1, false)); buffered_data.add(req_data_1); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(req_data_2, false)); diff --git a/test/extensions/filters/http/ext_proc/ordering_test.cc b/test/extensions/filters/http/ext_proc/ordering_test.cc index a12be075b034..b83cb3e3cbe5 100644 --- a/test/extensions/filters/http/ext_proc/ordering_test.cc +++ b/test/extensions/filters/http/ext_proc/ordering_test.cc @@ -492,13 +492,7 @@ TEST_F(OrderingTest, ResponseSomeDataComesFast) { EXPECT_CALL(stream_delegate_, send(_, false)); sendResponseHeaders(true); - // Some of the data might come back but we should watermark so that we - // don't fill the buffer. - EXPECT_CALL(encoder_callbacks_, onEncoderFilterAboveWriteBufferHighWatermark()); EXPECT_EQ(FilterDataStatus::StopIterationAndWatermark, filter_->encodeData(resp_body_1, false)); - - // When the response does comes back, we should lift the watermark - EXPECT_CALL(encoder_callbacks_, onEncoderFilterBelowWriteBufferLowWatermark()); sendResponseHeadersReply(); EXPECT_CALL(stream_delegate_, send(_, false)); diff --git a/test/extensions/filters/http/local_ratelimit/BUILD b/test/extensions/filters/http/local_ratelimit/BUILD index bb876c07f70c..e71f31db0857 100644 --- a/test/extensions/filters/http/local_ratelimit/BUILD +++ b/test/extensions/filters/http/local_ratelimit/BUILD @@ -22,8 +22,7 @@ envoy_extension_cc_test( "//source/extensions/filters/http/local_ratelimit:local_ratelimit_lib", "//test/common/stream_info:test_util", "//test/mocks/http:http_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/upstream:cluster_manager_mocks", + "//test/mocks/server:server_mocks", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/extensions/filters/http/local_ratelimit/v3:pkg_cc_proto", diff --git a/test/extensions/filters/http/local_ratelimit/filter_test.cc b/test/extensions/filters/http/local_ratelimit/filter_test.cc index 4495dcaf7a6a..cfecb31560b0 100644 --- a/test/extensions/filters/http/local_ratelimit/filter_test.cc +++ b/test/extensions/filters/http/local_ratelimit/filter_test.cc @@ -4,8 +4,7 @@ #include "source/extensions/filters/http/local_ratelimit/local_ratelimit.h" #include "test/mocks/http/mocks.h" -#include "test/mocks/local_info/mocks.h" -#include "test/mocks/upstream/cluster_manager.h" +#include "test/mocks/server/mocks.h" #include "test/test_common/test_runtime.h" #include "test/test_common/thread_factory_for_test.h" @@ -64,12 +63,12 @@ class FilterTest : public testing::Test { void setupPerRoute(const std::string& yaml, const bool enabled = true, const bool enforced = true, const bool per_route = false) { EXPECT_CALL( - runtime_.snapshot_, + factory_context_.runtime_loader_.snapshot_, featureEnabled(absl::string_view("test_enabled"), testing::Matcher(Percent(100)))) .WillRepeatedly(testing::Return(enabled)); EXPECT_CALL( - runtime_.snapshot_, + factory_context_.runtime_loader_.snapshot_, featureEnabled(absl::string_view("test_enforced"), testing::Matcher(Percent(100)))) .WillRepeatedly(testing::Return(enforced)); @@ -80,8 +79,7 @@ class FilterTest : public testing::Test { envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit config; TestUtility::loadFromYaml(yaml, config); config_ = - std::make_shared(config, local_info_, dispatcher_, cm_, singleton_manager_, - *stats_.rootScope(), runtime_, per_route); + std::make_shared(config, factory_context_, *stats_.rootScope(), per_route); filter_ = std::make_shared(config_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); @@ -103,10 +101,8 @@ class FilterTest : public testing::Test { testing::NiceMock decoder_callbacks_; testing::NiceMock decoder_callbacks_2_; NiceMock dispatcher_; - NiceMock runtime_; - NiceMock local_info_; - NiceMock cm_; - Singleton::ManagerImpl singleton_manager_; + + NiceMock factory_context_; std::shared_ptr config_; std::shared_ptr filter_; @@ -115,7 +111,7 @@ class FilterTest : public testing::Test { TEST_F(FilterTest, Runtime) { setup(fmt::format(config_yaml, "false", "1", "false", "\"OFF\""), false, false); - EXPECT_EQ(&runtime_, &(config_->runtime())); + EXPECT_EQ(&factory_context_.runtime_loader_, &(config_->runtime())); } TEST_F(FilterTest, ToErrorCode) { @@ -400,6 +396,56 @@ enable_x_ratelimit_headers: {} stage: {} )"; +static constexpr absl::string_view inlined_descriptor_config_yaml = R"( +stat_prefix: test +token_bucket: + max_tokens: {} + tokens_per_fill: 1 + fill_interval: 60s +filter_enabled: + runtime_key: test_enabled + default_value: + numerator: 100 + denominator: HUNDRED +filter_enforced: + runtime_key: test_enforced + default_value: + numerator: 100 + denominator: HUNDRED +response_headers_to_add: + - append_action: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-test-rate-limit + value: 'true' +enable_x_ratelimit_headers: {} +descriptors: +- entries: + - key: hello + value: world + - key: foo + value: bar + token_bucket: + max_tokens: 10 + tokens_per_fill: 10 + fill_interval: 60s +- entries: + - key: foo2 + value: bar2 + token_bucket: + max_tokens: {} + tokens_per_fill: 1 + fill_interval: 60s +rate_limits: +- actions: + - header_value_match: + descriptor_key: foo2 + descriptor_value: bar2 + headers: + - name: x-header-name + string_match: + exact: test_value + )"; + static constexpr absl::string_view consume_default_token_config_yaml = R"( stat_prefix: test token_bucket: @@ -938,6 +984,24 @@ TEST_F(DescriptorFilterTest, IncludeVirtualHostRateLimitsSetTrue) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); } +TEST_F(DescriptorFilterTest, UseInlinedRateLimitConfig) { + setUpTest(fmt::format(inlined_descriptor_config_yaml, "10", R"("OFF")", "1")); + + auto headers = Http::TestRequestHeaderMapImpl(); + // Requests will not be blocked because the requests don't match any descriptor and + // the global token bucket has enough tokens. + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + + headers.setCopy(Http::LowerCaseString("x-header-name"), "test_value"); + + // Only one request is allowed in 60s for the matched request. + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + EXPECT_CALL(decoder_callbacks_, sendLocalReply(Http::Code::TooManyRequests, _, _, _, _)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(headers, false)); +} + } // namespace LocalRateLimitFilter } // namespace HttpFilters } // namespace Extensions diff --git a/test/extensions/filters/http/rate_limit_quota/client_test.cc b/test/extensions/filters/http/rate_limit_quota/client_test.cc index 336b47b7fff5..2765f7e53c54 100644 --- a/test/extensions/filters/http/rate_limit_quota/client_test.cc +++ b/test/extensions/filters/http/rate_limit_quota/client_test.cc @@ -17,6 +17,8 @@ class RateLimitClientTest : public testing::Test { RateLimitTestClient test_client{}; }; +using envoy::service::rate_limit_quota::v3::RateLimitQuotaResponse; + TEST_F(RateLimitClientTest, OpenAndCloseStream) { EXPECT_OK(test_client.client_->startStream(&test_client.stream_info_)); EXPECT_CALL(test_client.stream_, closeStream()); @@ -53,7 +55,7 @@ TEST_F(RateLimitClientTest, SendRequestAndReceiveResponse) { // `onQuotaResponse` callback is expected to be called. EXPECT_CALL(test_client.callbacks_, onQuotaResponse); - envoy::service::rate_limit_quota::v3::RateLimitQuotaResponse resp; + RateLimitQuotaResponse resp; auto response_buf = Grpc::Common::serializeMessage(resp); EXPECT_TRUE(test_client.stream_callbacks_->onReceiveMessageRaw(std::move(response_buf))); @@ -93,6 +95,92 @@ TEST_F(RateLimitClientTest, RestartStreamWhileInUse) { { test_client.client_->sendUsageReport(bucket_id_hash); }); } +TEST_F(RateLimitClientTest, HandlingDuplicateTokenBucketAssignments) { + EXPECT_OK(test_client.client_->startStream(&test_client.stream_info_)); + ASSERT_NE(test_client.stream_callbacks_, nullptr); + + auto empty_request_headers = Http::RequestHeaderMapImpl::create(); + test_client.stream_callbacks_->onCreateInitialMetadata(*empty_request_headers); + auto empty_response_headers = Http::ResponseHeaderMapImpl::create(); + test_client.stream_callbacks_->onReceiveInitialMetadata(std::move(empty_response_headers)); + + // `onQuotaResponse` callback is expected to be called twice. + EXPECT_CALL(test_client.callbacks_, onQuotaResponse).Times(3); + + ::envoy::type::v3::TokenBucket token_bucket; + token_bucket.set_max_tokens(100); + token_bucket.mutable_tokens_per_fill()->set_value(10); + token_bucket.mutable_fill_interval()->set_seconds(1000); + + ::envoy::service::rate_limit_quota::v3::BucketId bucket_id; + bucket_id.mutable_bucket()->insert({"fairshare_group_id", "mock_group"}); + const size_t bucket_id_hash = MessageUtil::hash(bucket_id); + + Bucket initial_bucket_state; + initial_bucket_state.bucket_id = bucket_id; + test_client.bucket_cache_.insert( + {bucket_id_hash, std::make_unique(std::move(initial_bucket_state))}); + + RateLimitQuotaResponse::BucketAction action; + action.mutable_quota_assignment_action() + ->mutable_rate_limit_strategy() + ->mutable_token_bucket() + ->MergeFrom(token_bucket); + action.mutable_bucket_id()->MergeFrom(bucket_id); + + RateLimitQuotaResponse resp; + resp.add_bucket_action()->MergeFrom(action); + RateLimitQuotaResponse duplicate_resp; + duplicate_resp.add_bucket_action()->MergeFrom(action); + + auto response_buf = Grpc::Common::serializeMessage(resp); + auto duplicate_response_buf = Grpc::Common::serializeMessage(duplicate_resp); + EXPECT_TRUE(test_client.stream_callbacks_->onReceiveMessageRaw(std::move(response_buf))); + + ASSERT_EQ(test_client.bucket_cache_.size(), 1); + ASSERT_TRUE(test_client.bucket_cache_.contains(bucket_id_hash)); + Bucket* first_bucket = test_client.bucket_cache_.at(bucket_id_hash).get(); + TokenBucket* first_token_bucket_limiter = first_bucket->token_bucket_limiter.get(); + EXPECT_TRUE(first_token_bucket_limiter); + + // Send a duplicate response & expect the token bucket to be carried forward + // in the cache to avoid resetting token consumption. + EXPECT_TRUE( + test_client.stream_callbacks_->onReceiveMessageRaw(std::move(duplicate_response_buf))); + + ASSERT_EQ(test_client.bucket_cache_.size(), 1); + ASSERT_TRUE(test_client.bucket_cache_.contains(bucket_id_hash)); + Bucket* second_bucket = test_client.bucket_cache_.at(bucket_id_hash).get(); + TokenBucket* second_token_bucket_limiter = second_bucket->token_bucket_limiter.get(); + EXPECT_TRUE(second_token_bucket_limiter); + EXPECT_EQ(first_token_bucket_limiter, second_token_bucket_limiter); + + // Expect the limiter to be replaced if the config changes. + resp.mutable_bucket_action(0) + ->mutable_quota_assignment_action() + ->mutable_rate_limit_strategy() + ->mutable_token_bucket() + ->set_max_tokens(200); + auto different_response_buf = Grpc::Common::serializeMessage(resp); + EXPECT_TRUE( + test_client.stream_callbacks_->onReceiveMessageRaw(std::move(different_response_buf))); + + ASSERT_EQ(test_client.bucket_cache_.size(), 1); + ASSERT_TRUE(test_client.bucket_cache_.contains(bucket_id_hash)); + Bucket* third_bucket = test_client.bucket_cache_.at(bucket_id_hash).get(); + TokenBucket* third_token_bucket_limiter = third_bucket->token_bucket_limiter.get(); + EXPECT_TRUE(third_token_bucket_limiter); + EXPECT_NE(first_token_bucket_limiter, third_token_bucket_limiter); + + auto empty_response_trailers = Http::ResponseTrailerMapImpl::create(); + test_client.stream_callbacks_->onReceiveTrailingMetadata(std::move(empty_response_trailers)); + + EXPECT_CALL(test_client.stream_, closeStream()); + EXPECT_CALL(test_client.stream_, resetStream()); + test_client.client_->closeStream(); + test_client.client_->onRemoteClose(0, ""); +} + } // namespace } // namespace RateLimitQuota } // namespace HttpFilters diff --git a/test/extensions/filters/network/generic_proxy/access_log_test.cc b/test/extensions/filters/network/generic_proxy/access_log_test.cc index b0041c6d08dd..04426bdb0667 100644 --- a/test/extensions/filters/network/generic_proxy/access_log_test.cc +++ b/test/extensions/filters/network/generic_proxy/access_log_test.cc @@ -63,112 +63,114 @@ TEST(AccessLogFormatterTest, AccessLogFormatterTest) { { // Test for %METHOD%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%METHOD%"); + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create("%METHOD%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeRequest request; request.method_ = "FAKE_METHOD"; context.request_ = &request; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_METHOD"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_METHOD"); } { // Test for %HOST%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%HOST%"); + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create("%HOST%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeRequest request; request.host_ = "FAKE_HOST"; context.request_ = &request; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_HOST"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_HOST"); } { // Test for %PATH%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%PATH%"); + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create("%PATH%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeRequest request; request.path_ = "FAKE_PATH"; context.request_ = &request; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_PATH"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_PATH"); } { // Test for %PROTOCOL%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%PROTOCOL%"); + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create("%PROTOCOL%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeRequest request; request.protocol_ = "FAKE_PROTOCOL"; context.request_ = &request; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_PROTOCOL"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_PROTOCOL"); } { // Test for %REQUEST_PROPERTY%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%REQUEST_PROPERTY(FAKE_KEY)%"); + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create( + "%REQUEST_PROPERTY(FAKE_KEY)%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeRequest request; context.request_ = &request; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); request.data_["FAKE_KEY"] = "FAKE_VALUE"; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_VALUE"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_VALUE"); } { // Test for %RESPONSE_PROPERTY%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter( + auto formatter = *Envoy::Formatter::FormatterBaseImpl::create( "%RESPONSE_PROPERTY(FAKE_KEY)%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeResponse response; context.response_ = &response; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); response.data_["FAKE_KEY"] = "FAKE_VALUE"; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "FAKE_VALUE"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "FAKE_VALUE"); } { // Test for %GENERIC_RESPONSE_CODE%. FormatterContext context; - Envoy::Formatter::FormatterBaseImpl formatter("%GENERIC_RESPONSE_CODE%"); + auto formatter = + *Envoy::Formatter::FormatterBaseImpl::create("%GENERIC_RESPONSE_CODE%"); StreamInfo::MockStreamInfo stream_info; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-"); FakeStreamCodecFactory::FakeResponse response; response.status_ = {-1234, false}; context.response_ = &response; - EXPECT_EQ(formatter.formatWithContext(context, stream_info), "-1234"); + EXPECT_EQ(formatter->formatWithContext(context, stream_info), "-1234"); } } diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 9805d0f72c76..9c8aa06424e0 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -869,6 +869,29 @@ TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersKbMaxConfiguredViaRunti EXPECT_EQ(9000, config.maxRequestHeadersKb()); } +TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersCountMaxConfiguredViaRuntime) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + route_config: + name: local_route + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + )EOF"; + + ON_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + getInteger("envoy.reloadable_features.max_request_headers_count", _)) + .WillByDefault(Return(42)); + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + &scoped_routes_config_provider_manager_, tracer_manager_, + filter_config_provider_manager_, creation_status_); + ASSERT_TRUE(creation_status_.ok()); + EXPECT_EQ(42, config.maxRequestHeadersCount()); +} + // Validated that an explicit zero stream idle timeout disables. TEST_F(HttpConnectionManagerConfigTest, DisabledStreamIdleTimeout) { const std::string yaml_string = R"EOF( diff --git a/test/extensions/formatter/req_without_query/req_without_query_test.cc b/test/extensions/formatter/req_without_query/req_without_query_test.cc index e019ef4e9173..255f3df18409 100644 --- a/test/extensions/formatter/req_without_query/req_without_query_test.cc +++ b/test/extensions/formatter/req_without_query/req_without_query_test.cc @@ -156,9 +156,9 @@ TEST_F(ReqWithoutQueryTest, TestParserNotRecognizingCommand) { )EOF"; TestUtility::loadFromYaml(yaml, config_); - EXPECT_THROW(Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_) - .IgnoreError(), - EnvoyException); + EXPECT_FALSE(Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_) + .status() + .ok()); } } // namespace Formatter diff --git a/test/extensions/load_balancing_policies/client_side_weighted_round_robin/BUILD b/test/extensions/load_balancing_policies/client_side_weighted_round_robin/BUILD index e228a826408e..0a6739e131b5 100644 --- a/test/extensions/load_balancing_policies/client_side_weighted_round_robin/BUILD +++ b/test/extensions/load_balancing_policies/client_side_weighted_round_robin/BUILD @@ -42,6 +42,7 @@ envoy_extension_cc_test( size = "large", srcs = ["integration_test.cc"], extension_names = ["envoy.load_balancing_policies.client_side_weighted_round_robin"], + rbe_pool = "2core", deps = [ "//source/common/protobuf", "//source/extensions/load_balancing_policies/client_side_weighted_round_robin:config", diff --git a/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc b/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc index b2a830cd2173..2dcbc115d96b 100644 --- a/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc +++ b/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc @@ -2,6 +2,7 @@ #include "source/common/network/dns_resolver/dns_factory_util.h" #include "source/common/network/utility.h" +#include "source/extensions/network//dns_resolver/getaddrinfo/getaddrinfo.h" #include "test/mocks/api/mocks.h" #include "test/test_common/threadsafe_singleton_injector.h" @@ -9,6 +10,7 @@ #include "gtest/gtest.h" +using testing::ElementsAre; using testing::NiceMock; using testing::Return; @@ -104,18 +106,34 @@ class GetAddrInfoDnsImplTest : public testing::Test { DnsResolverSharedPtr resolver_; NiceMock os_sys_calls_; envoy::extensions::network::dns_resolver::getaddrinfo::v3::GetAddrInfoDnsResolverConfig config_; + ActiveDnsQuery* active_dns_query_; }; +MATCHER_P(HasTrace, expected_trace, "") { + std::vector v = absl::StrSplit(arg, '='); + uint8_t trace = std::stoi(v.at(0)); + return trace == static_cast(expected_trace); +} + TEST_F(GetAddrInfoDnsImplTest, LocalhostResolve) { // See https://github.com/envoyproxy/envoy/issues/28504. DISABLE_UNDER_WINDOWS; - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - verifyRealGaiResponse(status, std::move(response)); - dispatcher_->exit(); - }); + initialize(); + + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + verifyRealGaiResponse(status, std::move(response)); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } @@ -124,75 +142,110 @@ TEST_F(GetAddrInfoDnsImplTest, Cancel) { // See https://github.com/envoyproxy/envoy/issues/28504. DISABLE_UNDER_WINDOWS; + initialize(); + auto query = resolver_->resolve( "localhost", DnsLookupFamily::All, [](DnsResolver::ResolutionStatus, absl::string_view, std::list&&) { FAIL(); }); query->cancel(ActiveDnsQuery::CancelReason::QueryAbandoned); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - verifyRealGaiResponse(status, std::move(response)); - dispatcher_->exit(); - }); + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + verifyRealGaiResponse(status, std::move(response)); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } TEST_F(GetAddrInfoDnsImplTest, Failure) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); EXPECT_CALL(os_sys_calls_, getaddrinfo(_, _, _, _)) .WillOnce(Return(Api::SysCallIntResult{EAI_FAIL, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view details, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Failure); - EXPECT_EQ("Non-recoverable failure in name resolution", details); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view details, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Failure); + EXPECT_EQ("Non-recoverable failure in name resolution", details); + EXPECT_TRUE(response.empty()); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Failed), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } TEST_F(GetAddrInfoDnsImplTest, NoData) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); EXPECT_CALL(os_sys_calls_, getaddrinfo(_, _, _, _)) .WillOnce(Return(Api::SysCallIntResult{EAI_NODATA, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); + EXPECT_TRUE(response.empty()); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::NoResult), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } TEST_F(GetAddrInfoDnsImplTest, NoName) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); EXPECT_CALL(os_sys_calls_, getaddrinfo(_, _, _, _)) .WillOnce(Return(Api::SysCallIntResult{EAI_NONAME, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); + EXPECT_TRUE(response.empty()); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::NoResult), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } TEST_F(GetAddrInfoDnsImplTest, TryAgainIndefinitelyAndSuccess) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); // 2 calls - one EAGAIN, one success. @@ -200,14 +253,22 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainIndefinitelyAndSuccess) { .Times(2) .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{0, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = + resolver_->resolve("localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); + EXPECT_TRUE(response.empty()); + std::vector traces = + absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT(traces, ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Retrying), + HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } @@ -232,6 +293,8 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainThenCancel) { } TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndSuccess) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); config_.mutable_num_retries()->set_value(3); @@ -244,19 +307,30 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndSuccess) { .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{0, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = resolver_->resolve( + "localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); + EXPECT_TRUE(response.empty()); + std::vector traces = absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT( + traces, + ElementsAre(HasTrace(GetAddrInfoTrace::NotStarted), + HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), + HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), + HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Retrying), + HasTrace(GetAddrInfoTrace::Starting), HasTrace(GetAddrInfoTrace::Success), + HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndFailure) { + initialize(); + TestThreadsafeSingletonInjector os_calls(&os_sys_calls_); config_.mutable_num_retries()->set_value(3); @@ -269,15 +343,24 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndFailure) { .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})) .WillOnce(Return(Api::SysCallIntResult{EAI_AGAIN, 0})); - resolver_->resolve("localhost", DnsLookupFamily::All, - [this](DnsResolver::ResolutionStatus status, absl::string_view details, - std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Failure); - EXPECT_FALSE(details.empty()); - EXPECT_TRUE(response.empty()); - - dispatcher_->exit(); - }); + active_dns_query_ = resolver_->resolve( + "localhost", DnsLookupFamily::All, + [this](DnsResolver::ResolutionStatus status, absl::string_view details, + std::list&& response) { + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Failure); + EXPECT_FALSE(details.empty()); + EXPECT_TRUE(response.empty()); + std::vector traces = absl::StrSplit(active_dns_query_->getTraces(), ','); + EXPECT_THAT( + traces, + ElementsAre( + HasTrace(GetAddrInfoTrace::NotStarted), HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Retrying), HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Retrying), HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::Retrying), HasTrace(GetAddrInfoTrace::Starting), + HasTrace(GetAddrInfoTrace::DoneRetrying), HasTrace(GetAddrInfoTrace::Callback))); + dispatcher_->exit(); + }); dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } diff --git a/test/extensions/transport_sockets/http_11_proxy/connect_integration_test.cc b/test/extensions/transport_sockets/http_11_proxy/connect_integration_test.cc index 37ecf34ad0c0..3835e801f9f2 100644 --- a/test/extensions/transport_sockets/http_11_proxy/connect_integration_test.cc +++ b/test/extensions/transport_sockets/http_11_proxy/connect_integration_test.cc @@ -74,7 +74,7 @@ name: envoy.clusters.dynamic_forward_proxy bootstrap.mutable_static_resources()->mutable_clusters(0)->mutable_transport_socket(); envoy::config::core::v3::TransportSocket inner_socket; inner_socket.CopyFrom(*transport_socket); - if (inner_socket.name().empty()) { + if (set_inner_transport_socket_ && inner_socket.name().empty()) { inner_socket.set_name("envoy.transport_sockets.raw_buffer"); } transport_socket->set_name("envoy.transport_sockets.http_11_proxy"); @@ -135,12 +135,28 @@ name: envoy.clusters.dynamic_forward_proxy } bool use_alpn_ = false; bool try_http3_ = false; + + // If true, we'll explicitly set the inner "transport_socket" field to raw buffer if it is not + // configured. + bool set_inner_transport_socket_ = true; }; INSTANTIATE_TEST_SUITE_P(IpVersions, Http11ConnectHttpIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); +// Test that request/response is successful via plaintext if the inner transport socket is not set. +TEST_P(Http11ConnectHttpIntegrationTest, NoInnerTransportSocketSet) { + set_inner_transport_socket_ = false; + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = + sendRequestAndWaitForResponse(default_request_headers_, 0, default_response_headers_, 0); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); +} + // Test that with no connect-proxy header, the transport socket is a no-op. TEST_P(Http11ConnectHttpIntegrationTest, NoHeader) { initialize(); diff --git a/test/integration/BUILD b/test/integration/BUILD index 58a33e4a5215..374b9f7b3f89 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -798,13 +798,16 @@ envoy_cc_test( "//source/common/http:header_map_lib", "//source/extensions/filters/http/buffer:config", "//test/integration/filters:add_body_filter_config_lib", + "//test/integration/filters:add_encode_metadata_filter_lib", "//test/integration/filters:add_invalid_data_filter_lib", "//test/integration/filters:assert_non_reentrant_filter_lib", "//test/integration/filters:buffer_continue_filter_lib", "//test/integration/filters:continue_after_local_reply_filter_lib", "//test/integration/filters:continue_headers_only_inject_body", "//test/integration/filters:encoder_decoder_buffer_filter_lib", + "//test/integration/filters:encoder_recreate_stream_filter_lib", "//test/integration/filters:invalid_header_filter_lib", + "//test/integration/filters:local_reply_during_decoding_filter_lib", "//test/integration/filters:local_reply_during_encoding_data_filter_lib", "//test/integration/filters:local_reply_during_encoding_filter_lib", "//test/integration/filters:local_reply_with_metadata_filter_lib", diff --git a/test/integration/filter_integration_test.cc b/test/integration/filter_integration_test.cc index e83a9d231188..5028b5eca525 100644 --- a/test/integration/filter_integration_test.cc +++ b/test/integration/filter_integration_test.cc @@ -1575,5 +1575,85 @@ TEST_P(FilterIntegrationTest, FilterAddsDataToHeaderOnlyRequestWithIndependentHa testFilterAddsDataAndTrailersToHeaderOnlyRequest(); } +// Add metadata in the first filter before recreate the stream in the second filter, +// on response path. +TEST_P(FilterIntegrationTest, RecreateStreamAfterEncodeMetadata) { + // recreateStream is not supported in Upstream filter chain. + if (!testing_downstream_filter_) { + return; + } + + prependFilter("{ name: add-metadata-encode-headers-filter }"); + prependFilter("{ name: encoder-recreate-stream-filter }"); + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(default_response_headers_, true); + + // Second upstream request is triggered by recreateStream. + FakeStreamPtr upstream_request_2; + // Wait for the next stream on the upstream connection. + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_2)); + // Wait for the stream to be completely received. + ASSERT_TRUE(upstream_request_2->waitForEndStream(*dispatcher_)); + upstream_request_2->encodeHeaders(default_response_headers_, true); + + // Wait for the response to be completely received. + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + + // Verify the metadata is received. + std::set expected_metadata_keys = {"headers", "duplicate"}; + EXPECT_EQ(response->metadataMap().size(), expected_metadata_keys.size()); + for (const auto& key : expected_metadata_keys) { + // keys are the same as their corresponding values. + auto it = response->metadataMap().find(key); + ASSERT_FALSE(it == response->metadataMap().end()) << "key: " << key; + EXPECT_EQ(response->metadataMap().find(key)->second, key); + } +} + +// Add metadata in the first filter on local reply path. +TEST_P(FilterIntegrationTest, EncodeMetadataOnLocalReply) { + // Local replies are not seen by upstream HTTP filters. add-metadata-encode-headers-filter will + // not be invoked if it is installed in upstream filter chain. + // Thus, this test is only applicable to downstream filter chain. + if (!testing_downstream_filter_) { + return; + } + + prependFilter("{ name: local-reply-during-decode }"); + prependFilter("{ name: add-metadata-encode-headers-filter }"); + + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("500", response->headers().getStatusValue()); + + // Verify the metadata is received. + std::set expected_metadata_keys = {"headers", "duplicate"}; + EXPECT_EQ(response->metadataMap().size(), expected_metadata_keys.size()); + for (const auto& key : expected_metadata_keys) { + // keys are the same as their corresponding values. + auto it = response->metadataMap().find(key); + ASSERT_FALSE(it == response->metadataMap().end()) << "key: " << key; + EXPECT_EQ(response->metadataMap().find(key)->second, key); + } +} + } // namespace } // namespace Envoy diff --git a/test/integration/filters/BUILD b/test/integration/filters/BUILD index 35e062867d71..6dd174614bdd 100644 --- a/test/integration/filters/BUILD +++ b/test/integration/filters/BUILD @@ -1013,3 +1013,36 @@ envoy_cc_test_library( "//test/extensions/filters/http/common:empty_http_filter_config_lib", ], ) + +envoy_cc_test_library( + name = "add_encode_metadata_filter_lib", + srcs = [ + "add_encode_metadata_filter.cc", + ], + deps = [ + ":common_lib", + "//envoy/event:timer_interface", + "//envoy/http:filter_interface", + "//envoy/registry", + "//envoy/server:filter_config_interface", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "//test/extensions/filters/http/common:empty_http_filter_config_lib", + ], +) + +envoy_cc_test_library( + name = "encoder_recreate_stream_filter_lib", + srcs = [ + "encoder_recreate_stream_filter.cc", + ], + deps = [ + ":common_lib", + "//envoy/event:timer_interface", + "//envoy/http:filter_interface", + "//envoy/registry", + "//envoy/server:filter_config_interface", + "//source/common/router:string_accessor_lib", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "//test/extensions/filters/http/common:empty_http_filter_config_lib", + ], +) diff --git a/test/integration/filters/add_encode_metadata_filter.cc b/test/integration/filters/add_encode_metadata_filter.cc new file mode 100644 index 000000000000..5f56ac39b005 --- /dev/null +++ b/test/integration/filters/add_encode_metadata_filter.cc @@ -0,0 +1,41 @@ +#include +#include + +#include "envoy/event/timer.h" +#include "envoy/http/filter.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" + +#include "test/extensions/filters/http/common/empty_http_filter_config.h" +#include "test/integration/filters/common.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +// A filter add encoded metadata in encodeHeaders. +class AddEncodeMetadataFilter : public Http::PassThroughFilter { +public: + constexpr static char name[] = "add-metadata-encode-headers-filter"; + + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap&, bool) override { + Http::MetadataMap metadata_map = {{"headers", "headers"}, {"duplicate", "duplicate"}}; + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); + return Http::FilterHeadersStatus::Continue; + } + + Http::FilterDataStatus encodeData(Buffer::Instance&, bool) override { + return Http::FilterDataStatus::Continue; + } +}; + +constexpr char AddEncodeMetadataFilter::name[]; +static Registry::RegisterFactory, + Server::Configuration::NamedHttpFilterConfigFactory> + register_; + +} // namespace Envoy diff --git a/test/integration/filters/encoder_recreate_stream_filter.cc b/test/integration/filters/encoder_recreate_stream_filter.cc new file mode 100644 index 000000000000..ebcbb4e40ed9 --- /dev/null +++ b/test/integration/filters/encoder_recreate_stream_filter.cc @@ -0,0 +1,55 @@ +#include +#include + +#include "envoy/event/timer.h" +#include "envoy/http/filter.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/common/router/string_accessor_impl.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" + +#include "test/extensions/filters/http/common/empty_http_filter_config.h" +#include "test/integration/filters/common.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +class EncoderRecreateStreamFilter : public Http::PassThroughFilter { +public: + constexpr static char name[] = "encoder-recreate-stream-filter"; + + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap&, bool) override { + const auto* filter_state = + decoder_callbacks_->streamInfo().filterState()->getDataReadOnly( + "test_key"); + + if (filter_state != nullptr) { + return ::Envoy::Http::FilterHeadersStatus::Continue; + } + + decoder_callbacks_->streamInfo().filterState()->setData( + "test_key", std::make_unique("test_value"), + StreamInfo::FilterState::StateType::ReadOnly, StreamInfo::FilterState::LifeSpan::Request); + + if (decoder_callbacks_->recreateStream(nullptr)) { + return ::Envoy::Http::FilterHeadersStatus::StopIteration; + } + + return ::Envoy::Http::FilterHeadersStatus::Continue; + } + + void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override { + decoder_callbacks_ = &callbacks; + } +}; + +// perform static registration +constexpr char EncoderRecreateStreamFilter::name[]; +static Registry::RegisterFactory, + Server::Configuration::NamedHttpFilterConfigFactory> + register_; + +} // namespace Envoy diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index bf8dea48a423..e53403354a0c 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -4975,10 +4975,10 @@ TEST_P(ProtocolIntegrationTest, InvalidResponseHeaderNameStreamError) { EXPECT_EQ("502", response->headers().getStatusValue()); test_server_->waitForCounterGe("http.config_test.downstream_rq_5xx", 1); - std::string error_message = - upstreamProtocol() == Http::CodecType::HTTP3 - ? "upstream_reset_before_response_started{protocol_error|QUIC_BAD_APPLICATION_PAYLOAD}" - : "upstream_reset_before_response_started{protocol_error}"; + std::string error_message = upstreamProtocol() == Http::CodecType::HTTP3 + ? "upstream_reset_before_response_started{protocol_error|QUIC_" + "BAD_APPLICATION_PAYLOAD|FROM_SELF}" + : "upstream_reset_before_response_started{protocol_error}"; EXPECT_EQ(waitForAccessLog(access_log_name_), error_message); // Upstream connection should stay up diff --git a/test/integration/xds_config_tracker_integration_test.cc b/test/integration/xds_config_tracker_integration_test.cc index 9838ef2edf65..b80eae8717fc 100644 --- a/test/integration/xds_config_tracker_integration_test.cc +++ b/test/integration/xds_config_tracker_integration_test.cc @@ -68,15 +68,14 @@ class TestXdsConfigTracker : public Config::XdsConfigTracker { } } - void onConfigAccepted( - const absl::string_view, - const Protobuf::RepeatedPtrField& resources, - const Protobuf::RepeatedPtrField&) override { + void onConfigAccepted(const absl::string_view, + absl::Span resources, + const Protobuf::RepeatedPtrField&) override { stats_.on_config_accepted_.inc(); test::envoy::config::xds::TestTrackerMetadata test_metadata; - for (const auto& resource : resources) { - if (resource.has_metadata()) { - const auto& config_typed_metadata = resource.metadata().typed_filter_metadata(); + for (const auto* resource : resources) { + if (resource->has_metadata()) { + const auto& config_typed_metadata = resource->metadata().typed_filter_metadata(); if (const auto& metadata_it = config_typed_metadata.find(kTestKey); metadata_it != config_typed_metadata.end()) { const auto status = Envoy::MessageUtil::unpackTo(metadata_it->second, test_metadata); diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index f153a22aa229..85a10b89a3da 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -56,11 +56,10 @@ class MockUntypedConfigUpdateCallbacks : public UntypedConfigUpdateCallbacks { MOCK_METHOD(void, onConfigUpdate, (const std::vector& resources, const std::string& version_info)); - MOCK_METHOD( - void, onConfigUpdate, - (const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info)); + MOCK_METHOD(void, onConfigUpdate, + (absl::Span added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info)); MOCK_METHOD(void, onConfigUpdateFailed, (Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e)); }; @@ -131,6 +130,13 @@ class MockGrpcMux : public GrpcMux { MOCK_METHOD(bool, paused, (const std::string& type_url), (const)); MOCK_METHOD(EdsResourcesCacheOptRef, edsResourcesCache, ()); + + MOCK_METHOD(absl::Status, updateMuxSource, + (Grpc::RawAsyncClientPtr && primary_async_client, + Grpc::RawAsyncClientPtr&& failover_async_client, + CustomConfigValidatorsPtr&& custom_config_validators, Stats::Scope& scope, + BackOffStrategyPtr&& backoff_strategy, + const envoy::config::core::v3::ApiConfigSource& ads_config_source)); }; class MockGrpcStreamCallbacks diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 7174b21667f6..c7260c34a8c9 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -43,6 +43,8 @@ class MockActiveDnsQuery : public ActiveDnsQuery { // Network::ActiveDnsQuery MOCK_METHOD(void, cancel, (CancelReason reason)); + MOCK_METHOD(void, addTrace, (uint8_t)); + MOCK_METHOD(std::string, getTraces, ()); }; class MockFilterManager : public FilterManager { diff --git a/test/server/admin/admin_instance.cc b/test/server/admin/admin_instance.cc index a2b4d2f15d17..919a1b3083df 100644 --- a/test/server/admin/admin_instance.cc +++ b/test/server/admin/admin_instance.cc @@ -14,7 +14,7 @@ AdminInstanceTest::AdminInstanceTest() std::list access_logs; Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, "/dev/null"}; access_logs.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( - file_info, {}, Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + file_info, {}, *Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), server_.accessLogManager())); server_.options_.admin_address_path_ = TestEnvironment::temporaryPath("admin.address"); admin_.startHttpListener(access_logs, Network::Test::getCanonicalLoopbackAddress(GetParam()), diff --git a/test/server/admin/admin_test.cc b/test/server/admin/admin_test.cc index fb6f64ff6d22..997e7900ca80 100644 --- a/test/server/admin/admin_test.cc +++ b/test/server/admin/admin_test.cc @@ -82,7 +82,7 @@ TEST_P(AdminInstanceTest, AdminAddress) { std::list access_logs; Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, "/dev/null"}; access_logs.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( - file_info, {}, Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + file_info, {}, *Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), server_.accessLogManager())); EXPECT_LOG_CONTAINS( "info", "admin address:", @@ -97,7 +97,7 @@ TEST_P(AdminInstanceTest, AdminBadAddressOutPath) { std::list access_logs; Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, "/dev/null"}; access_logs.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( - file_info, {}, Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), + file_info, {}, *Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), server_.accessLogManager())); EXPECT_LOG_CONTAINS( "critical", diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 505741eccdaf..99c5b90a40e1 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -92,98 +92,98 @@ aiohappyeyeballs==2.4.0 \ --hash=sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2 \ --hash=sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd # via aiohttp -aiohttp==3.10.6 \ - --hash=sha256:02108326574ff60267b7b35b17ac5c0bbd0008ccb942ce4c48b657bb90f0b8aa \ - --hash=sha256:029a019627b37fa9eac5c75cc54a6bb722c4ebbf5a54d8c8c0fb4dd8facf2702 \ - --hash=sha256:03fa40d1450ee5196e843315ddf74a51afc7e83d489dbfc380eecefea74158b1 \ - --hash=sha256:0749c4d5a08a802dd66ecdf59b2df4d76b900004017468a7bb736c3b5a3dd902 \ - --hash=sha256:0754690a3a26e819173a34093798c155bafb21c3c640bff13be1afa1e9d421f9 \ - --hash=sha256:0a75d5c9fb4f06c41d029ae70ad943c3a844c40c0a769d12be4b99b04f473d3d \ - --hash=sha256:0b82c8ebed66ce182893e7c0b6b60ba2ace45b1df104feb52380edae266a4850 \ - --hash=sha256:0be3115753baf8b4153e64f9aa7bf6c0c64af57979aa900c31f496301b374570 \ - --hash=sha256:14477c4e52e2f17437b99893fd220ffe7d7ee41df5ebf931a92b8ca82e6fd094 \ - --hash=sha256:164ecd32e65467d86843dbb121a6666c3deb23b460e3f8aefdcaacae79eb718a \ - --hash=sha256:1cb045ec5961f51af3e2c08cd6fe523f07cc6e345033adee711c49b7b91bb954 \ - --hash=sha256:1e52e59ed5f4cc3a3acfe2a610f8891f216f486de54d95d6600a2c9ba1581f4d \ - --hash=sha256:217791c6a399cc4f2e6577bb44344cba1f5714a2aebf6a0bea04cfa956658284 \ - --hash=sha256:25d92f794f1332f656e3765841fc2b7ad5c26c3f3d01e8949eeb3495691cf9f4 \ - --hash=sha256:2708baccdc62f4b1251e59c2aac725936a900081f079b88843dabcab0feeeb27 \ - --hash=sha256:27cf19a38506e2e9f12fc17e55f118f04897b0a78537055d93a9de4bf3022e3d \ - --hash=sha256:289fa8a20018d0d5aa9e4b35d899bd51bcb80f0d5f365d9a23e30dac3b79159b \ - --hash=sha256:2cd5290ab66cfca2f90045db2cc6434c1f4f9fbf97c9f1c316e785033782e7d2 \ - --hash=sha256:2dd56e3c43660ed3bea67fd4c5025f1ac1f9ecf6f0b991a6e5efe2e678c490c5 \ - --hash=sha256:3427031064b0d5c95647e6369c4aa3c556402f324a3e18107cb09517abe5f962 \ - --hash=sha256:3468b39f977a11271517c6925b226720e148311039a380cc9117b1e2258a721f \ - --hash=sha256:370e2d47575c53c817ee42a18acc34aad8da4dbdaac0a6c836d58878955f1477 \ - --hash=sha256:3d2665c5df629eb2f981dab244c01bfa6cdc185f4ffa026639286c4d56fafb54 \ - --hash=sha256:3e15e33bfc73fa97c228f72e05e8795e163a693fd5323549f49367c76a6e5883 \ - --hash=sha256:3fb4216e3ec0dbc01db5ba802f02ed78ad8f07121be54eb9e918448cc3f61b7c \ - --hash=sha256:40271a2a375812967401c9ca8077de9368e09a43a964f4dce0ff603301ec9358 \ - --hash=sha256:438c5863feb761f7ca3270d48c292c334814459f61cc12bab5ba5b702d7c9e56 \ - --hash=sha256:4407a80bca3e694f2d2a523058e20e1f9f98a416619e04f6dc09dc910352ac8b \ - --hash=sha256:444d1704e2af6b30766debed9be8a795958029e552fe77551355badb1944012c \ - --hash=sha256:4611db8c907f90fe86be112efdc2398cd7b4c8eeded5a4f0314b70fdea8feab0 \ - --hash=sha256:473961b3252f3b949bb84873d6e268fb6d8aa0ccc6eb7404fa58c76a326bb8e1 \ - --hash=sha256:4752df44df48fd42b80f51d6a97553b482cda1274d9dc5df214a3a1aa5d8f018 \ - --hash=sha256:47647c8af04a70e07a2462931b0eba63146a13affa697afb4ecbab9d03a480ce \ - --hash=sha256:482f74057ea13d387a7549d7a7ecb60e45146d15f3e58a2d93a0ad2d5a8457cd \ - --hash=sha256:4bef1480ee50f75abcfcb4b11c12de1005968ca9d0172aec4a5057ba9f2b644f \ - --hash=sha256:4fabdcdc781a36b8fd7b2ca9dea8172f29a99e11d00ca0f83ffeb50958da84a1 \ - --hash=sha256:5582de171f0898139cf51dd9fcdc79b848e28d9abd68e837f0803fc9f30807b1 \ - --hash=sha256:58c5d7318a136a3874c78717dd6de57519bc64f6363c5827c2b1cb775bea71dd \ - --hash=sha256:5db26bbca8e7968c4c977a0c640e0b9ce7224e1f4dcafa57870dc6ee28e27de6 \ - --hash=sha256:614fc21e86adc28e4165a6391f851a6da6e9cbd7bb232d0df7718b453a89ee98 \ - --hash=sha256:6419728b08fb6380c66a470d2319cafcec554c81780e2114b7e150329b9a9a7f \ - --hash=sha256:669c0efe7e99f6d94d63274c06344bd0e9c8daf184ce5602a29bc39e00a18720 \ - --hash=sha256:66bc81361131763660b969132a22edce2c4d184978ba39614e8f8f95db5c95f8 \ - --hash=sha256:671745ea7db19693ce867359d503772177f0b20fa8f6ee1e74e00449f4c4151d \ - --hash=sha256:682836fc672972cc3101cc9e30d49c5f7e8f1d010478d46119fe725a4545acfd \ - --hash=sha256:6a504d7cdb431a777d05a124fd0b21efb94498efa743103ea01b1e3136d2e4fb \ - --hash=sha256:6a86610174de8a85a920e956e2d4f9945e7da89f29a00e95ac62a4a414c4ef4e \ - --hash=sha256:6b50b367308ca8c12e0b50cba5773bc9abe64c428d3fd2bbf5cd25aab37c77bf \ - --hash=sha256:7475da7a5e2ccf1a1c86c8fee241e277f4874c96564d06f726d8df8e77683ef7 \ - --hash=sha256:7641920bdcc7cd2d3ddfb8bb9133a6c9536b09dbd49490b79e125180b2d25b93 \ - --hash=sha256:79a9f42efcc2681790595ab3d03c0e52d01edc23a0973ea09f0dc8d295e12b8e \ - --hash=sha256:7ea35d849cdd4a9268f910bff4497baebbc1aa3f2f625fd8ccd9ac99c860c621 \ - --hash=sha256:8198b7c002aae2b40b2d16bfe724b9a90bcbc9b78b2566fc96131ef4e382574d \ - --hash=sha256:81b292f37969f9cc54f4643f0be7dacabf3612b3b4a65413661cf6c350226787 \ - --hash=sha256:844d48ff9173d0b941abed8b2ea6a412f82b56d9ab1edb918c74000c15839362 \ - --hash=sha256:8617c96a20dd57e7e9d398ff9d04f3d11c4d28b1767273a5b1a018ada5a654d3 \ - --hash=sha256:8a637d387db6fdad95e293fab5433b775fd104ae6348d2388beaaa60d08b38c4 \ - --hash=sha256:92351aa5363fc3c1f872ca763f86730ced32b01607f0c9662b1fa711087968d0 \ - --hash=sha256:9843d683b8756971797be171ead21511d2215a2d6e3c899c6e3107fbbe826791 \ - --hash=sha256:995ab1a238fd0d19dc65f2d222e5eb064e409665c6426a3e51d5101c1979ee84 \ - --hash=sha256:9bd6b2033993d5ae80883bb29b83fb2b432270bbe067c2f53cc73bb57c46065f \ - --hash=sha256:9d26da22a793dfd424be1050712a70c0afd96345245c29aced1e35dbace03413 \ - --hash=sha256:a976ef488f26e224079deb3d424f29144c6d5ba4ded313198169a8af8f47fb82 \ - --hash=sha256:a9f196c970db2dcde4f24317e06615363349dc357cf4d7a3b0716c20ac6d7bcd \ - --hash=sha256:b169f8e755e541b72e714b89a831b315bbe70db44e33fead28516c9e13d5f931 \ - --hash=sha256:b504c08c45623bf5c7ca41be380156d925f00199b3970efd758aef4a77645feb \ - --hash=sha256:ba18573bb1de1063d222f41de64a0d3741223982dcea863b3f74646faf618ec7 \ - --hash=sha256:ba3662d41abe2eab0eeec7ee56f33ef4e0b34858f38abf24377687f9e1fb00a5 \ - --hash=sha256:bd294dcdc1afdc510bb51d35444003f14e327572877d016d576ac3b9a5888a27 \ - --hash=sha256:bdbeff1b062751c2a2a55b171f7050fb7073633c699299d042e962aacdbe1a07 \ - --hash=sha256:bf861da9a43d282d6dd9dcd64c23a0fccf2c5aa5cd7c32024513c8c79fb69de3 \ - --hash=sha256:c82a94ddec996413a905f622f3da02c4359952aab8d817c01cf9915419525e95 \ - --hash=sha256:c91781d969fbced1993537f45efe1213bd6fccb4b37bfae2a026e20d6fbed206 \ - --hash=sha256:c9721cdd83a994225352ca84cd537760d41a9da3c0eacb3ff534747ab8fba6d0 \ - --hash=sha256:cca776a440795db437d82c07455761c85bbcf3956221c3c23b8c93176c278ce7 \ - --hash=sha256:cf8b8560aa965f87bf9c13bf9fed7025993a155ca0ce8422da74bf46d18c2f5f \ - --hash=sha256:d2578ef941be0c2ba58f6f421a703527d08427237ed45ecb091fed6f83305336 \ - --hash=sha256:d2b3935a22c9e41a8000d90588bed96cf395ef572dbb409be44c6219c61d900d \ - --hash=sha256:d4dfa5ad4bce9ca30a76117fbaa1c1decf41ebb6c18a4e098df44298941566f9 \ - --hash=sha256:d7f408c43f5e75ea1edc152fb375e8f46ef916f545fb66d4aebcbcfad05e2796 \ - --hash=sha256:dc1a16f3fc1944c61290d33c88dc3f09ba62d159b284c38c5331868425aca426 \ - --hash=sha256:e0009258e97502936d3bd5bf2ced15769629097d0abb81e6495fba1047824fe0 \ - --hash=sha256:e05b39158f2af0e2438cc2075cfc271f4ace0c3cc4a81ec95b27a0432e161951 \ - --hash=sha256:e1f80cd17d81a404b6e70ef22bfe1870bafc511728397634ad5f5efc8698df56 \ - --hash=sha256:e2e7d5591ea868d5ec82b90bbeb366a198715672841d46281b623e23079593db \ - --hash=sha256:f3af26f86863fad12e25395805bb0babbd49d512806af91ec9708a272b696248 \ - --hash=sha256:f52e54fd776ad0da1006708762213b079b154644db54bcfc62f06eaa5b896402 \ - --hash=sha256:f8b8e49fe02f744d38352daca1dbef462c3874900bd8166516f6ea8e82b5aacf \ - --hash=sha256:fb138fbf9f53928e779650f5ed26d0ea1ed8b2cab67f0ea5d63afa09fdc07593 \ - --hash=sha256:fe517113fe4d35d9072b826c3e147d63c5f808ca8167d450b4f96c520c8a1d8d \ - --hash=sha256:ff99ae06eef85c7a565854826114ced72765832ee16c7e3e766c5e4c5b98d20e +aiohttp==3.10.9 \ + --hash=sha256:02d1d6610588bcd743fae827bd6f2e47e0d09b346f230824b4c6fb85c6065f9c \ + --hash=sha256:03690541e4cc866eef79626cfa1ef4dd729c5c1408600c8cb9e12e1137eed6ab \ + --hash=sha256:0bc059ecbce835630e635879f5f480a742e130d9821fbe3d2f76610a6698ee25 \ + --hash=sha256:0c21c82df33b264216abffff9f8370f303dab65d8eee3767efbbd2734363f677 \ + --hash=sha256:1298b854fd31d0567cbb916091be9d3278168064fca88e70b8468875ef9ff7e7 \ + --hash=sha256:1321658f12b6caffafdc35cfba6c882cb014af86bef4e78c125e7e794dfb927b \ + --hash=sha256:143b0026a9dab07a05ad2dd9e46aa859bffdd6348ddc5967b42161168c24f857 \ + --hash=sha256:16e6a51d8bc96b77f04a6764b4ad03eeef43baa32014fce71e882bd71302c7e4 \ + --hash=sha256:172ad884bb61ad31ed7beed8be776eb17e7fb423f1c1be836d5cb357a096bf12 \ + --hash=sha256:17c272cfe7b07a5bb0c6ad3f234e0c336fb53f3bf17840f66bd77b5815ab3d16 \ + --hash=sha256:1a0ee6c0d590c917f1b9629371fce5f3d3f22c317aa96fbdcce3260754d7ea21 \ + --hash=sha256:2746d8994ebca1bdc55a1e998feff4e94222da709623bb18f6e5cfec8ec01baf \ + --hash=sha256:2914caa46054f3b5ff910468d686742ff8cff54b8a67319d75f5d5945fd0a13d \ + --hash=sha256:2bbf94d4a0447705b7775417ca8bb8086cc5482023a6e17cdc8f96d0b1b5aba6 \ + --hash=sha256:2bd9f3eac515c16c4360a6a00c38119333901b8590fe93c3257a9b536026594d \ + --hash=sha256:2c33fa6e10bb7ed262e3ff03cc69d52869514f16558db0626a7c5c61dde3c29f \ + --hash=sha256:2d37f4718002863b82c6f391c8efd4d3a817da37030a29e2682a94d2716209de \ + --hash=sha256:3668d0c2a4d23fb136a753eba42caa2c0abbd3d9c5c87ee150a716a16c6deec1 \ + --hash=sha256:36d4fba838be5f083f5490ddd281813b44d69685db910907636bc5dca6322316 \ + --hash=sha256:40ff5b7660f903dc587ed36ef08a88d46840182d9d4b5694e7607877ced698a1 \ + --hash=sha256:42775de0ca04f90c10c5c46291535ec08e9bcc4756f1b48f02a0657febe89b10 \ + --hash=sha256:482c85cf3d429844396d939b22bc2a03849cb9ad33344689ad1c85697bcba33a \ + --hash=sha256:4e6cb75f8ddd9c2132d00bc03c9716add57f4beff1263463724f6398b813e7eb \ + --hash=sha256:4edc3fd701e2b9a0d605a7b23d3de4ad23137d23fc0dbab726aa71d92f11aaaf \ + --hash=sha256:4fd16b30567c5b8e167923be6e027eeae0f20cf2b8a26b98a25115f28ad48ee0 \ + --hash=sha256:5002a02c17fcfd796d20bac719981d2fca9c006aac0797eb8f430a58e9d12431 \ + --hash=sha256:51d0a4901b27272ae54e42067bc4b9a90e619a690b4dc43ea5950eb3070afc32 \ + --hash=sha256:558b3d223fd631ad134d89adea876e7fdb4c93c849ef195049c063ada82b7d08 \ + --hash=sha256:5c070430fda1a550a1c3a4c2d7281d3b8cfc0c6715f616e40e3332201a253067 \ + --hash=sha256:5f392ef50e22c31fa49b5a46af7f983fa3f118f3eccb8522063bee8bfa6755f8 \ + --hash=sha256:60555211a006d26e1a389222e3fab8cd379f28e0fbf7472ee55b16c6c529e3a6 \ + --hash=sha256:608cecd8d58d285bfd52dbca5b6251ca8d6ea567022c8a0eaae03c2589cd9af9 \ + --hash=sha256:60ad5b8a7452c0f5645c73d4dad7490afd6119d453d302cd5b72b678a85d6044 \ + --hash=sha256:63649309da83277f06a15bbdc2a54fbe75efb92caa2c25bb57ca37762789c746 \ + --hash=sha256:6ebdc3b3714afe1b134b3bbeb5f745eed3ecbcff92ab25d80e4ef299e83a5465 \ + --hash=sha256:6f3c6648aa123bcd73d6f26607d59967b607b0da8ffcc27d418a4b59f4c98c7c \ + --hash=sha256:7003f33f5f7da1eb02f0446b0f8d2ccf57d253ca6c2e7a5732d25889da82b517 \ + --hash=sha256:776e9f3c9b377fcf097c4a04b241b15691e6662d850168642ff976780609303c \ + --hash=sha256:85711eec2d875cd88c7eb40e734c4ca6d9ae477d6f26bd2b5bb4f7f60e41b156 \ + --hash=sha256:87d1e4185c5d7187684d41ebb50c9aeaaaa06ca1875f4c57593071b0409d2444 \ + --hash=sha256:8a3f063b41cc06e8d0b3fcbbfc9c05b7420f41287e0cd4f75ce0a1f3d80729e6 \ + --hash=sha256:8b3fb28a9ac8f2558760d8e637dbf27aef1e8b7f1d221e8669a1074d1a266bb2 \ + --hash=sha256:8bd9125dd0cc8ebd84bff2be64b10fdba7dc6fd7be431b5eaf67723557de3a31 \ + --hash=sha256:8be1a65487bdfc285bd5e9baf3208c2132ca92a9b4020e9f27df1b16fab998a9 \ + --hash=sha256:8cc0d13b4e3b1362d424ce3f4e8c79e1f7247a00d792823ffd640878abf28e56 \ + --hash=sha256:8d9d10d10ec27c0d46ddaecc3c5598c4db9ce4e6398ca872cdde0525765caa2f \ + --hash=sha256:8debb45545ad95b58cc16c3c1cc19ad82cffcb106db12b437885dbee265f0ab5 \ + --hash=sha256:91aa966858593f64c8a65cdefa3d6dc8fe3c2768b159da84c1ddbbb2c01ab4ef \ + --hash=sha256:9331dd34145ff105177855017920dde140b447049cd62bb589de320fd6ddd582 \ + --hash=sha256:99f9678bf0e2b1b695e8028fedac24ab6770937932eda695815d5a6618c37e04 \ + --hash=sha256:9fdf5c839bf95fc67be5794c780419edb0dbef776edcfc6c2e5e2ffd5ee755fa \ + --hash=sha256:a14e4b672c257a6b94fe934ee62666bacbc8e45b7876f9dd9502d0f0fe69db16 \ + --hash=sha256:a19caae0d670771ea7854ca30df76f676eb47e0fd9b2ee4392d44708f272122d \ + --hash=sha256:a35ed3d03910785f7d9d6f5381f0c24002b2b888b298e6f941b2fc94c5055fcd \ + --hash=sha256:a61df62966ce6507aafab24e124e0c3a1cfbe23c59732987fc0fd0d71daa0b88 \ + --hash=sha256:a6e00c8a92e7663ed2be6fcc08a2997ff06ce73c8080cd0df10cc0321a3168d7 \ + --hash=sha256:ac3196952c673822ebed8871cf8802e17254fff2a2ed4835d9c045d9b88c5ec7 \ + --hash=sha256:ac74e794e3aee92ae8f571bfeaa103a141e409863a100ab63a253b1c53b707eb \ + --hash=sha256:ad3675c126f2a95bde637d162f8231cff6bc0bc9fbe31bd78075f9ff7921e322 \ + --hash=sha256:aeebd3061f6f1747c011e1d0b0b5f04f9f54ad1a2ca183e687e7277bef2e0da2 \ + --hash=sha256:ba1a599255ad6a41022e261e31bc2f6f9355a419575b391f9655c4d9e5df5ff5 \ + --hash=sha256:bbdb8def5268f3f9cd753a265756f49228a20ed14a480d151df727808b4531dd \ + --hash=sha256:c2555e4949c8d8782f18ef20e9d39730d2656e218a6f1a21a4c4c0b56546a02e \ + --hash=sha256:c2695c61cf53a5d4345a43d689f37fc0f6d3a2dc520660aec27ec0f06288d1f9 \ + --hash=sha256:c2b627d3c8982691b06d89d31093cee158c30629fdfebe705a91814d49b554f8 \ + --hash=sha256:c46131c6112b534b178d4e002abe450a0a29840b61413ac25243f1291613806a \ + --hash=sha256:c54dc329cd44f7f7883a9f4baaefe686e8b9662e2c6c184ea15cceee587d8d69 \ + --hash=sha256:c7d7cafc11d70fdd8801abfc2ff276744ae4cb39d8060b6b542c7e44e5f2cfc2 \ + --hash=sha256:cb0b2d5d51f96b6cc19e6ab46a7b684be23240426ae951dcdac9639ab111b45e \ + --hash=sha256:d15a29424e96fad56dc2f3abed10a89c50c099f97d2416520c7a543e8fddf066 \ + --hash=sha256:d1f5c9169e26db6a61276008582d945405b8316aae2bb198220466e68114a0f5 \ + --hash=sha256:d271f770b52e32236d945911b2082f9318e90ff835d45224fa9e28374303f729 \ + --hash=sha256:d646fdd74c25bbdd4a055414f0fe32896c400f38ffbdfc78c68e62812a9e0257 \ + --hash=sha256:d6e395c3d1f773cf0651cd3559e25182eb0c03a2777b53b4575d8adc1149c6e9 \ + --hash=sha256:d7c071235a47d407b0e93aa6262b49422dbe48d7d8566e1158fecc91043dd948 \ + --hash=sha256:d97273a52d7f89a75b11ec386f786d3da7723d7efae3034b4dda79f6f093edc1 \ + --hash=sha256:dcf354661f54e6a49193d0b5653a1b011ba856e0b7a76bda2c33e4c6892f34ea \ + --hash=sha256:e3e7fabedb3fe06933f47f1538df7b3a8d78e13d7167195f51ca47ee12690373 \ + --hash=sha256:e525b69ee8a92c146ae5b4da9ecd15e518df4d40003b01b454ad694a27f498b5 \ + --hash=sha256:e709d6ac598c5416f879bb1bae3fd751366120ac3fa235a01de763537385d036 \ + --hash=sha256:e83dfefb4f7d285c2d6a07a22268344a97d61579b3e0dce482a5be0251d672ab \ + --hash=sha256:e86260b76786c28acf0b5fe31c8dca4c2add95098c709b11e8c35b424ebd4f5b \ + --hash=sha256:e883b61b75ca6efc2541fcd52a5c8ccfe288b24d97e20ac08fdf343b8ac672ea \ + --hash=sha256:f0a44bb40b6aaa4fb9a5c1ee07880570ecda2065433a96ccff409c9c20c1624a \ + --hash=sha256:f82ace0ec57c94aaf5b0e118d4366cff5889097412c75aa14b4fd5fc0c44ee3e \ + --hash=sha256:f9ca09414003c0e96a735daa1f071f7d7ed06962ef4fa29ceb6c80d06696d900 \ + --hash=sha256:fa430b871220dc62572cef9c69b41e0d70fcb9d486a4a207a5de4c1f25d82593 \ + --hash=sha256:fc262c3df78c8ff6020c782d9ce02e4bcffe4900ad71c0ecdad59943cba54442 \ + --hash=sha256:fcd546782d03181b0b1d20b43d612429a90a68779659ba8045114b867971ab71 \ + --hash=sha256:fd4ceeae2fb8cabdd1b71c82bfdd39662473d3433ec95b962200e9e752fb70d0 \ + --hash=sha256:fec5fac7aea6c060f317f07494961236434928e6f4374e170ef50b3001e14581 # via # -r requirements.in # aio-api-github @@ -586,40 +586,45 @@ flake8==7.1.1 \ # -r requirements.in # envoy-code-check # pep8-naming -frozendict==2.4.4 \ - --hash=sha256:07c3a5dee8bbb84cba770e273cdbf2c87c8e035903af8f781292d72583416801 \ - --hash=sha256:12a342e439aef28ccec533f0253ea53d75fe9102bd6ea928ff530e76eac38906 \ - --hash=sha256:1697793b5f62b416c0fc1d94638ec91ed3aa4ab277f6affa3a95216ecb3af170 \ - --hash=sha256:199a4d32194f3afed6258de7e317054155bc9519252b568d9cfffde7e4d834e5 \ - --hash=sha256:259528ba6b56fa051bc996f1c4d8b57e30d6dd3bc2f27441891b04babc4b5e73 \ - --hash=sha256:2b70b431e3a72d410a2cdf1497b3aba2f553635e0c0f657ce311d841bf8273b6 \ - --hash=sha256:2bd009cf4fc47972838a91e9b83654dc9a095dc4f2bb3a37c3f3124c8a364543 \ - --hash=sha256:2d8536e068d6bf281f23fa835ac07747fb0f8851879dd189e9709f9567408b4d \ - --hash=sha256:3148062675536724502c6344d7c485dd4667fdf7980ca9bd05e338ccc0c4471e \ - --hash=sha256:3f7c031b26e4ee6a3f786ceb5e3abf1181c4ade92dce1f847da26ea2c96008c7 \ - --hash=sha256:4297d694eb600efa429769125a6f910ec02b85606f22f178bafbee309e7d3ec7 \ - --hash=sha256:4a59578d47b3949437519b5c39a016a6116b9e787bb19289e333faae81462e59 \ - --hash=sha256:4ae8d05c8d0b6134bfb6bfb369d5fa0c4df21eabb5ca7f645af95fdc6689678e \ - --hash=sha256:5d58d9a8d9e49662c6dafbea5e641f97decdb3d6ccd76e55e79818415362ba25 \ - --hash=sha256:63aa49f1919af7d45fb8fd5dec4c0859bc09f46880bd6297c79bb2db2969b63d \ - --hash=sha256:6874fec816b37b6eb5795b00e0574cba261bf59723e2de607a195d5edaff0786 \ - --hash=sha256:6eb716e6a6d693c03b1d53280a1947716129f5ef9bcdd061db5c17dea44b80fe \ - --hash=sha256:705efca8d74d3facbb6ace80ab3afdd28eb8a237bfb4063ed89996b024bc443d \ - --hash=sha256:78c94991944dd33c5376f720228e5b252ee67faf3bac50ef381adc9e51e90d9d \ - --hash=sha256:7f79c26dff10ce11dad3b3627c89bb2e87b9dd5958c2b24325f16a23019b8b94 \ - --hash=sha256:7fee9420475bb6ff357000092aa9990c2f6182b2bab15764330f4ad7de2eae49 \ - --hash=sha256:812ab17522ba13637826e65454115a914c2da538356e85f43ecea069813e4b33 \ - --hash=sha256:85375ec6e979e6373bffb4f54576a68bf7497c350861d20686ccae38aab69c0a \ - --hash=sha256:87ebcde21565a14fe039672c25550060d6f6d88cf1f339beac094c3b10004eb0 \ - --hash=sha256:93a7b19afb429cbf99d56faf436b45ef2fa8fe9aca89c49eb1610c3bd85f1760 \ - --hash=sha256:b3b967d5065872e27b06f785a80c0ed0a45d1f7c9b85223da05358e734d858ca \ - --hash=sha256:c6bf9260018d653f3cab9bd147bd8592bf98a5c6e338be0491ced3c196c034a3 \ - --hash=sha256:c8f92425686323a950337da4b75b4c17a3327b831df8c881df24038d560640d4 \ - --hash=sha256:d13b4310db337f4d2103867c5a05090b22bc4d50ca842093779ef541ea9c9eea \ - --hash=sha256:d9647563e76adb05b7cde2172403123380871360a114f546b4ae1704510801e5 \ - --hash=sha256:dc2228874eacae390e63fd4f2bb513b3144066a977dc192163c9f6c7f6de6474 \ - --hash=sha256:e1b941132d79ce72d562a13341d38fc217bc1ee24d8c35a20d754e79ff99e038 \ - --hash=sha256:fefeb700bc7eb8b4c2dc48704e4221860d254c8989fb53488540bc44e44a1ac2 +frozendict==2.4.5 \ + --hash=sha256:043bce9f5e7e8df8313d0c961b2da3346c0b002311d88c74cadae2b1f8fd2904 \ + --hash=sha256:0483776fa25f0c3f9ff75a89b5276a582231f83baea85b091dfefde99b95ac5c \ + --hash=sha256:05786d9cff61ee95149c15cdfc10cf4b4cb10e1eab1d4648b0c99ed58fedb86d \ + --hash=sha256:08e8d4ebc950916b5cdeedcb697cf893c5903d9cb8ab27dffe41072a08e1cfc1 \ + --hash=sha256:14755127b988b428f95f11fa626374e247f8a40316697218f1e2f44c107c5027 \ + --hash=sha256:1f364cfe5ef97523a4434e9f458bb4821594d3531d898621e5acae43463dcb5e \ + --hash=sha256:20cf1db30ca0c8f76a05ec1256132f505cf6f93eb382a5961c738377037b5b9d \ + --hash=sha256:24953a1cfe344415e7557413e493e21fa9c4cecbc13284b6f334837aa5601c9f \ + --hash=sha256:2b51e0aae859fc8d56138eac4be121a76647fa66ea620af8a62d9d6938729441 \ + --hash=sha256:35730ff269b8a6eb07c3d91d5fd6a7f5c32e08bef665fe1ef51012bdb2702196 \ + --hash=sha256:3702438d81deab127963ab41dca1738a0fbf11038a50a25c5f814943a2d25de5 \ + --hash=sha256:3af1b09bad761d5500b096b757c346591b5dfdc8443aa9642c2c02b4885dc09e \ + --hash=sha256:4b3eea1678607174c468fe4e3b903625bccfb3215ae2b0138220dc1f6ef71f37 \ + --hash=sha256:4f073c926b1f88fa85ed85222101f61f6c4b2180c95d1528ca6ecc7cef835442 \ + --hash=sha256:6be054ba76e8a49c6846a7369db3eaaa0593c29a644026c25923f13fdea06483 \ + --hash=sha256:6d10ad91ded20fae8737fddcfed1b6358207f95292a335f96c4cd365d97f5e30 \ + --hash=sha256:73ff80ce3f95c3ec60b38ef4ddf5cb8e20ea3edce2f56e89af7c4c45013684bf \ + --hash=sha256:754924d53b1fd2dae7b15f9c86b0610334a840ae728cc717f24aa040fc94136f \ + --hash=sha256:76c7af5f9db9ae01531edaf38fd3e8faae1bb6b94abbf8fa5bd0326f52e777df \ + --hash=sha256:8582f26ca862bb1e40ed790d934adf6afcfc904b0425dcfe01aed2103bed27fb \ + --hash=sha256:95bf788d65e3a50d9deef5c68e568c59acf473fea8a7a4d7e405a7e63ac85cdd \ + --hash=sha256:98b1483980c47bb74ef777ab748031a69eb41c43ef44494e9db79111638fdb84 \ + --hash=sha256:a5448639058157ccd7f26ba83e441066c6ae515beb68b9b99d5b2dbb0bb36b19 \ + --hash=sha256:a81d80ded07f0e34a3fa2b78235a48259b886ad20a96da5526a9404d192c6eb3 \ + --hash=sha256:a967730a115cb8b5d028e0d07cd680c04324e913ee3d1caef9e039ca740343c4 \ + --hash=sha256:af4bdffecff350a68f1a966dec84459d3ea8e2606c14fb3ebe937b8c6c9e9129 \ + --hash=sha256:b8481d83a7219e9e14c719113ea2d8321d7840af35af58c72b3864b7123e7de3 \ + --hash=sha256:bbf9dcb69a44519308c775bcf72b8417c3102f415f77df19b8eddb97b8f88d78 \ + --hash=sha256:d2b2a9c809ebb54c8c0fe2acc38d45299f3e9a9d90061773468896221cab1eaa \ + --hash=sha256:de1126d6dd39d7939be388161b973ffadb9c6e97f8f9fdcb52ba790828621191 \ + --hash=sha256:dee64c44e1a146171bdd2bf4501210f273a5ab73cc2e924a01f81b14e4612f94 \ + --hash=sha256:e591124818f2d5d8f2d284a7b78604f4180e73d770c33252370edccff5969293 \ + --hash=sha256:eb58e0c41bc6bb60a8b0852d50da8836380fb78311608dc26791e3e9aa25bcfe \ + --hash=sha256:ee3e6f4013446d17c580ce0bac41a1f81371c21dd6e34a85c0f26bae94fdfd9b \ + --hash=sha256:f1385852cb05a31ae76c972caa3a07679414b01a664848bde2bae71c9910f045 \ + --hash=sha256:f2ff03ed21657cde7ea04fffc76f0c1736100a03671e4682722685e63486cd96 \ + --hash=sha256:f8d677a6066ca3289181028fccabf77413addddaa7a6629485d52aeaa4f4899c \ + --hash=sha256:fd7add309789595c044c0155a0bddfa9d20c77f65de1e33a14aa3033b936ef63 # via # -r requirements.in # aio-run-runner @@ -764,9 +769,9 @@ humanfriendly==10.0 \ --hash=sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477 \ --hash=sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc # via coloredlogs -icalendar==5.0.13 \ - --hash=sha256:5ded5415e2e1edef5ab230024a75878a7a81d518a3b1ae4f34bf20b173c84dc2 \ - --hash=sha256:92799fde8cce0b61daa8383593836d1e19136e504fa1671f471f98be9b029706 +icalendar==6.0.0 \ + --hash=sha256:567e718551d800362db04ca09777295336e1803f6fc6bc0a7a5e258917fa8ed0 \ + --hash=sha256:7ddf60d343f3c1f716de9b62f6e80ffd95d03cab62464894a0539feab7b5c76e # via -r requirements.in idna==3.10 \ --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ @@ -790,9 +795,9 @@ jinja2==3.1.4 \ # envoy-base-utils # envoy-dependency-check # sphinx -kafka-python-ng==2.2.2 \ - --hash=sha256:3fab1a03133fade1b6fd5367ff726d980e59031c4aaca9bf02c516840a4f8406 \ - --hash=sha256:87ad3a766e2c0bec71d9b99bdd9e9c5cda62d96cfda61a8ca16510484d6ad7d4 +kafka-python-ng==2.2.3 \ + --hash=sha256:adc6e82147c441ca4ae1f22e291fc08efab0d10971cbd4aa1481d2ffa38e9480 \ + --hash=sha256:f79f28e10ade9b5a9860b2ec15b7cc8dc510d5702f5a399430478cff5f93a05a # via -r requirements.in markupsafe==2.1.5 \ --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ @@ -1179,7 +1184,6 @@ pytz==2024.2 \ # via # aio-core # envoy-base-utils - # icalendar pyu2f==0.1.5 \ --hash=sha256:a3caa3a11842fc7d5746376f37195e6af5f17c0a15737538bb1cebf656fb306b # via google-reauth @@ -1380,6 +1384,10 @@ typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via pygithub +tzdata==2024.2 \ + --hash=sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc \ + --hash=sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd + # via icalendar uritemplate==4.1.1 \ --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \ --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e @@ -1515,99 +1523,99 @@ yapf==0.40.2 \ # via # -r requirements.in # envoy-code-check -yarl==1.13.0 \ - --hash=sha256:01953b5686e5868fd0d8eaea4e484482c158597b8ddb9d9d4d048303fa3334c7 \ - --hash=sha256:01d3941d416e71ce65f33393beb50e93c1c9e8e516971b6653c96df6eb599a2c \ - --hash=sha256:02f117a63d11c8c2ada229029f8bb444a811e62e5041da962de548f26ac2c40f \ - --hash=sha256:0675a9cf65176e11692b20a516d5f744849251aa24024f422582d2d1bf7c8c82 \ - --hash=sha256:08037790f973367431b9406a7b9d940e872cca12e081bce3b7cea068daf81f0a \ - --hash=sha256:08a3b0b8d10092dade46424fe775f2c9bc32e5a985fdd6afe410fe28598db6b2 \ - --hash=sha256:08d15aff3477fecb7a469d1fdf5939a686fbc5a16858022897d3e9fc99301f19 \ - --hash=sha256:0b489858642e4e92203941a8fdeeb6373c0535aa986200b22f84d4b39cd602ba \ - --hash=sha256:0bd3caf554a52da78ec08415ebedeb6b9636436ca2afda9b5b9ff4a533478940 \ - --hash=sha256:0db15ce35dfd100bc9ab40173f143fbea26c84d7458d63206934fe5548fae25d \ - --hash=sha256:1129737da2291c9952a93c015e73583dd66054f3ae991c8674f6e39c46d95dd3 \ - --hash=sha256:12c92576633027f297c26e52aba89f6363b460f483d85cf7c14216eb55d06d02 \ - --hash=sha256:13a9cd39e47ca4dc25139d3c63fe0dc6acf1b24f9d94d3b5197ac578fbfd84bf \ - --hash=sha256:144b9e9164f21da81731c970dbda52245b343c0f67f3609d71013dd4d0db9ebf \ - --hash=sha256:1932c7bfa537f89ad5ca3d1e7e05de3388bb9e893230a384159fb974f6e9f90c \ - --hash=sha256:24b78a1f57780eeeb17f5e1be851ab9fa951b98811e1bb4b5a53f74eec3e2666 \ - --hash=sha256:26214b0a9b8f4b7b04e67eee94a82c9b4e5c721f4d1ce7e8c87c78f0809b7684 \ - --hash=sha256:2842a89b697d8ca3dda6a25b4e4d835d14afe25a315c8a79dbdf5f70edfd0960 \ - --hash=sha256:2cec7b52903dcf9008311167036775346dcb093bb15ed7ec876debc3095e7dab \ - --hash=sha256:2d7a44ae252efb0fcd79ac0997416721a44345f53e5aec4a24f489d983aa00e3 \ - --hash=sha256:2f6f4a352d0beea5dd39315ab16fc26f0122d13457a7e65ad4f06c7961dcf87a \ - --hash=sha256:31748bee7078db26008bf94d39693c682a26b5c3a80a67194a4c9c8fe3b5cf47 \ - --hash=sha256:33e2f5ef965e69a1f2a1b0071a70c4616157da5a5478f3c2f6e185e06c56a322 \ - --hash=sha256:3590ed9c7477059aea067a58ec87b433bbd47a2ceb67703b1098cca1ba075f0d \ - --hash=sha256:3628e4e572b1db95285a72c4be102356f2dfc6214d9f126de975fd51b517ae55 \ - --hash=sha256:37049eb26d637a5b2f00562f65aad679f5d231c4c044edcd88320542ad66a2d9 \ - --hash=sha256:38a3b742c923fe2cab7d2e2c67220d17da8d0433e8bfe038356167e697ef5524 \ - --hash=sha256:3a9b2650425b2ab9cc68865978963601b3c2414e1d94ef04f193dd5865e1bd79 \ - --hash=sha256:3ff602aa84420b301c083ae7f07df858ae8e371bf3be294397bda3e0b27c6290 \ - --hash=sha256:419c22b419034b4ee3ba1c27cbbfef01ca8d646f9292f614f008093143334cdc \ - --hash=sha256:4483680e129b2a4250be20947b554cd5f7140fa9e5a1e4f1f42717cf91f8676a \ - --hash=sha256:49bee8c99586482a238a7b2ec0ef94e5f186bfdbb8204d14a3dd31867b3875ce \ - --hash=sha256:4c73e0f8375b75806b8771890580566a2e6135e6785250840c4f6c45b69eb72d \ - --hash=sha256:4ffd8a9758b5df7401a49d50e76491f4c582cf7350365439563062cdff45bf16 \ - --hash=sha256:517f9d90ca0224bb7002266eba6e70d8fcc8b1d0c9321de2407e41344413ed46 \ - --hash=sha256:52b7bb09bb48f7855d574480e2efc0c30d31cab4e6ffc6203e2f7ffbf2e4496a \ - --hash=sha256:5378cb60f4209505f6aa60423c174336bd7b22e0d8beb87a2a99ad50787f1341 \ - --hash=sha256:5540b4896b244a6539f22b613b32b5d1b737e08011aa4ed56644cb0519d687df \ - --hash=sha256:561a5f6c054927cf5a993dd7b032aeebc10644419e65db7dd6bdc0b848806e65 \ - --hash=sha256:580fdb2ea48a40bcaa709ee0dc71f64e7a8f23b44356cc18cd9ce55dc3bc3212 \ - --hash=sha256:6072ff51eeb7938ecac35bf24fc465be00e75217eaa1ffad3cc7620accc0f6f4 \ - --hash=sha256:612bd8d2267558bea36347e4e6e3a96f436bdc5c011f1437824be4f2e3abc5e1 \ - --hash=sha256:66c028066be36d54e7a0a38e832302b23222e75db7e65ed862dc94effc8ef062 \ - --hash=sha256:73777f145cd591e1377bf8d8a97e5f8e39c9742ad0f100c898bba1f963aef662 \ - --hash=sha256:784d6e50ea96b3bbb078eb7b40d8c0e3674c2f12da4f0061f889b2cfdbab8f37 \ - --hash=sha256:79de5f8432b53d1261d92761f71dfab5fc7e1c75faa12a3535c27e681dacfa9d \ - --hash=sha256:801fb5dfc05910cd5ef4806726e2129d8c9a16cdfa26a8166697da0861e59dfc \ - --hash=sha256:8986fa2be78193dc8b8c27bd0d3667fe612f7232844872714c4200499d5225ca \ - --hash=sha256:8a67f20e97462dee8a89e9b997a78932959d2ed991e8f709514cb4160143e7b1 \ - --hash=sha256:8ab16c9e94726fdfcbf5b37a641c9d9d0b35cc31f286a2c3b9cad6451cb53b2b \ - --hash=sha256:91251614cca1ba4ab0507f1ba5f5a44e17a5e9a4c7f0308ea441a994bdac3fc7 \ - --hash=sha256:92a26956d268ad52bd2329c2c674890fe9e8669b41d83ed136e7037b1a29808e \ - --hash=sha256:92abbe37e3fb08935e0e95ac5f83f7b286a6f2575f542225ec7afde405ed1fa1 \ - --hash=sha256:9671d0d65f86e0a0eee59c5b05e381c44e3d15c36c2a67da247d5d82875b4e4e \ - --hash=sha256:9d2845f1a37438a8e11e4fcbbf6ffd64bc94dc9cb8c815f72d0eb6f6c622deb0 \ - --hash=sha256:a9a1a600e8449f3a24bc7dca513be8d69db173fe842e8332a7318b5b8757a6af \ - --hash=sha256:aa187a8599e0425f26b25987d884a8b67deb5565f1c450c3a6e8d3de2cdc8715 \ - --hash=sha256:aaf10e525e461f43831d82149d904f35929d89f3ccd65beaf7422aecd500dd39 \ - --hash=sha256:ab3ee57b25ce15f79ade27b7dfb5e678af26e4b93be5a4e22655acd9d40b81ba \ - --hash=sha256:acf27399c94270103d68f86118a183008d601e4c2c3a7e98dcde0e3b0163132f \ - --hash=sha256:acf8c219a59df22609cfaff4a7158a0946f273e3b03a5385f1fdd502496f0cff \ - --hash=sha256:b536c2ac042add7f276d4e5857b08364fc32f28e02add153f6f214de50f12d07 \ - --hash=sha256:b9648e5ae280babcac867b16e845ce51ed21f8c43bced2ca40cff7eee983d6d4 \ - --hash=sha256:bcb374db7a609484941c01380c1450728ec84d9c3e68cd9a5feaecb52626c4be \ - --hash=sha256:be828e92ae67a21d6a252aecd65668dddbf3bb5d5278660be607647335001119 \ - --hash=sha256:bf7a9b31729b97985d4a796808859dfd0e37b55f1ca948d46a568e56e51dd8fb \ - --hash=sha256:bff0d468664cdf7b2a6bfd5e17d4a7025edb52df12e0e6e17223387b421d425c \ - --hash=sha256:c2518660bd8166e770b76ce92514b491b8720ae7e7f5f975cd888b1592894d2c \ - --hash=sha256:c7d35ff2a5a51bc6d40112cdb4ca3fd9636482ce8c6ceeeee2301e34f7ed7556 \ - --hash=sha256:ca71238af0d247d07747cb7202a9359e6e1d6d9e277041e1ad2d9f36b3a111a6 \ - --hash=sha256:cdcdd49136d423ee5234c9360eae7063d3120a429ee984d7d9da821c012da4d7 \ - --hash=sha256:cf4f3a87bd52f8f33b0155cd0f6f22bdf2092d88c6c6acbb1aee3bc206ecbe35 \ - --hash=sha256:d04ea92a3643a9bb28aa6954fff718342caab2cc3d25d0160fe16e26c4a9acb7 \ - --hash=sha256:d42227711a4180d0c22cec30fd81d263d7bb378389d8e70b5f4c597e8abae202 \ - --hash=sha256:d78ebad57152d301284761b03a708aeac99c946a64ba967d47cbcc040e36688b \ - --hash=sha256:d807417ceebafb7ce18085a1205d28e8fcb1435a43197d7aa3fab98f5bfec5ef \ - --hash=sha256:d95fcc9508390db73a0f1c7e78d9a1b1a3532a3f34ceff97c0b3b04140fbe6e4 \ - --hash=sha256:db463fce425f935eee04a9182c74fdf9ed90d3bd2079d4a17f8fb7a2d7c11009 \ - --hash=sha256:db90702060b1cdb7c7609d04df5f68a12fd5581d013ad379e58e0c2e651d92b8 \ - --hash=sha256:de97ee57e00a82ebb8c378fc73c5d9a773e4c2cec8079ff34ebfef61c8ba5b11 \ - --hash=sha256:deb70c006be548076d628630aad9a3ef3a1b2c28aaa14b395cf0939b9124252e \ - --hash=sha256:e3b4293f02129cc2f5068f3687ef294846a79c9d19fabaa9bfdfeeebae11c001 \ - --hash=sha256:e480a12cec58009eeaeee7f48728dc8f629f8e0f280d84957d42c361969d84da \ - --hash=sha256:e4dddf99a853b3f60f3ce6363fb1ad94606113743cf653f116a38edd839a4461 \ - --hash=sha256:e5462756fb34c884ca9d4875b6d2ec80957a767123151c467c97a9b423617048 \ - --hash=sha256:e557e2681b47a0ecfdfbea44743b3184d94d31d5ce0e4b13ff64ce227a40f86e \ - --hash=sha256:ebb2236f8098205f59774a28e25a84601a4beb3e974157d418ee6c470d73e0dc \ - --hash=sha256:f3ef76df654f3547dcb76ba550f9ca59826588eecc6bd7df16937c620df32060 \ - --hash=sha256:f603216d62e9680bfac7fb168ef9673fd98abbb50c43e73d97615dfa1afebf57 \ - --hash=sha256:f997004ff530b5381290e82b212a93bd420fefe5a605872dc16fc7e4a7f4251e \ - --hash=sha256:fda4404bbb6f91e327827f4483d52fe24f02f92de91217954cf51b1cb9ee9c41 \ - --hash=sha256:fe6946c3cbcfbed67c5e50dae49baff82ad054aaa10ff7a4db8dfac646b7b479 +yarl==1.13.1 \ + --hash=sha256:08d7148ff11cb8e886d86dadbfd2e466a76d5dd38c7ea8ebd9b0e07946e76e4b \ + --hash=sha256:098b870c18f1341786f290b4d699504e18f1cd050ed179af8123fd8232513424 \ + --hash=sha256:11b3ca8b42a024513adce810385fcabdd682772411d95bbbda3b9ed1a4257644 \ + --hash=sha256:1891d69a6ba16e89473909665cd355d783a8a31bc84720902c5911dbb6373465 \ + --hash=sha256:1bbb418f46c7f7355084833051701b2301092e4611d9e392360c3ba2e3e69f88 \ + --hash=sha256:1d0828e17fa701b557c6eaed5edbd9098eb62d8838344486248489ff233998b8 \ + --hash=sha256:1d8e3ca29f643dd121f264a7c89f329f0fcb2e4461833f02de6e39fef80f89da \ + --hash=sha256:1fa56f34b2236f5192cb5fceba7bbb09620e5337e0b6dfe2ea0ddbd19dd5b154 \ + --hash=sha256:216a6785f296169ed52cd7dcdc2612f82c20f8c9634bf7446327f50398732a51 \ + --hash=sha256:22b739f99c7e4787922903f27a892744189482125cc7b95b747f04dd5c83aa9f \ + --hash=sha256:2430cf996113abe5aee387d39ee19529327205cda975d2b82c0e7e96e5fdabdc \ + --hash=sha256:269c201bbc01d2cbba5b86997a1e0f73ba5e2f471cfa6e226bcaa7fd664b598d \ + --hash=sha256:298c1eecfd3257aa16c0cb0bdffb54411e3e831351cd69e6b0739be16b1bdaa8 \ + --hash=sha256:2a93a4557f7fc74a38ca5a404abb443a242217b91cd0c4840b1ebedaad8919d4 \ + --hash=sha256:2b2442a415a5f4c55ced0fade7b72123210d579f7d950e0b5527fc598866e62c \ + --hash=sha256:2db874dd1d22d4c2c657807562411ffdfabec38ce4c5ce48b4c654be552759dc \ + --hash=sha256:309c104ecf67626c033845b860d31594a41343766a46fa58c3309c538a1e22b2 \ + --hash=sha256:31497aefd68036d8e31bfbacef915826ca2e741dbb97a8d6c7eac66deda3b606 \ + --hash=sha256:373f16f38721c680316a6a00ae21cc178e3a8ef43c0227f88356a24c5193abd6 \ + --hash=sha256:396e59b8de7e4d59ff5507fb4322d2329865b909f29a7ed7ca37e63ade7f835c \ + --hash=sha256:3bb83a0f12701c0b91112a11148b5217617982e1e466069d0555be9b372f2734 \ + --hash=sha256:3de86547c820e4f4da4606d1c8ab5765dd633189791f15247706a2eeabc783ae \ + --hash=sha256:3fdbf0418489525231723cdb6c79e7738b3cbacbaed2b750cb033e4ea208f220 \ + --hash=sha256:40c6e73c03a6befb85b72da213638b8aaa80fe4136ec8691560cf98b11b8ae6e \ + --hash=sha256:44a4c40a6f84e4d5955b63462a0e2a988f8982fba245cf885ce3be7618f6aa7d \ + --hash=sha256:44b07e1690f010c3c01d353b5790ec73b2f59b4eae5b0000593199766b3f7a5c \ + --hash=sha256:45d23c4668d4925688e2ea251b53f36a498e9ea860913ce43b52d9605d3d8177 \ + --hash=sha256:45f209fb4bbfe8630e3d2e2052535ca5b53d4ce2d2026bed4d0637b0416830da \ + --hash=sha256:4afdf84610ca44dcffe8b6c22c68f309aff96be55f5ea2fa31c0c225d6b83e23 \ + --hash=sha256:4feaaa4742517eaceafcbe74595ed335a494c84634d33961214b278126ec1485 \ + --hash=sha256:576365c9f7469e1f6124d67b001639b77113cfd05e85ce0310f5f318fd02fe85 \ + --hash=sha256:5820bd4178e6a639b3ef1db8b18500a82ceab6d8b89309e121a6859f56585b05 \ + --hash=sha256:5989a38ba1281e43e4663931a53fbf356f78a0325251fd6af09dd03b1d676a09 \ + --hash=sha256:5a9bacedbb99685a75ad033fd4de37129449e69808e50e08034034c0bf063f99 \ + --hash=sha256:5b66c87da3c6da8f8e8b648878903ca54589038a0b1e08dde2c86d9cd92d4ac9 \ + --hash=sha256:5c5e32fef09ce101fe14acd0f498232b5710effe13abac14cd95de9c274e689e \ + --hash=sha256:658e8449b84b92a4373f99305de042b6bd0d19bf2080c093881e0516557474a5 \ + --hash=sha256:6a2acde25be0cf9be23a8f6cbd31734536a264723fca860af3ae5e89d771cd71 \ + --hash=sha256:6a5185ad722ab4dd52d5fb1f30dcc73282eb1ed494906a92d1a228d3f89607b0 \ + --hash=sha256:6b7f6e699304717fdc265a7e1922561b02a93ceffdaefdc877acaf9b9f3080b8 \ + --hash=sha256:703b0f584fcf157ef87816a3c0ff868e8c9f3c370009a8b23b56255885528f10 \ + --hash=sha256:7055bbade838d68af73aea13f8c86588e4bcc00c2235b4b6d6edb0dbd174e246 \ + --hash=sha256:78f271722423b2d4851cf1f4fa1a1c4833a128d020062721ba35e1a87154a049 \ + --hash=sha256:7addd26594e588503bdef03908fc207206adac5bd90b6d4bc3e3cf33a829f57d \ + --hash=sha256:81bad32c8f8b5897c909bf3468bf601f1b855d12f53b6af0271963ee67fff0d2 \ + --hash=sha256:82e692fb325013a18a5b73a4fed5a1edaa7c58144dc67ad9ef3d604eccd451ad \ + --hash=sha256:84bbcdcf393139f0abc9f642bf03f00cac31010f3034faa03224a9ef0bb74323 \ + --hash=sha256:86c438ce920e089c8c2388c7dcc8ab30dfe13c09b8af3d306bcabb46a053d6f7 \ + --hash=sha256:8be8cdfe20787e6a5fcbd010f8066227e2bb9058331a4eccddec6c0db2bb85b2 \ + --hash=sha256:8c723c91c94a3bc8033dd2696a0f53e5d5f8496186013167bddc3fb5d9df46a3 \ + --hash=sha256:8ca53632007c69ddcdefe1e8cbc3920dd88825e618153795b57e6ebcc92e752a \ + --hash=sha256:8f722f30366474a99745533cc4015b1781ee54b08de73260b2bbe13316079851 \ + --hash=sha256:942c80a832a79c3707cca46bd12ab8aa58fddb34b1626d42b05aa8f0bcefc206 \ + --hash=sha256:94a993f976cdcb2dc1b855d8b89b792893220db8862d1a619efa7451817c836b \ + --hash=sha256:95c6737f28069153c399d875317f226bbdea939fd48a6349a3b03da6829fb550 \ + --hash=sha256:9915300fe5a0aa663c01363db37e4ae8e7c15996ebe2c6cce995e7033ff6457f \ + --hash=sha256:9a18595e6a2ee0826bf7dfdee823b6ab55c9b70e8f80f8b77c37e694288f5de1 \ + --hash=sha256:9c8854b9f80693d20cec797d8e48a848c2fb273eb6f2587b57763ccba3f3bd4b \ + --hash=sha256:9cec42a20eae8bebf81e9ce23fb0d0c729fc54cf00643eb251ce7c0215ad49fe \ + --hash=sha256:9d2e1626be8712333a9f71270366f4a132f476ffbe83b689dd6dc0d114796c74 \ + --hash=sha256:9d74f3c335cfe9c21ea78988e67f18eb9822f5d31f88b41aec3a1ec5ecd32da5 \ + --hash=sha256:9fb4134cc6e005b99fa29dbc86f1ea0a298440ab6b07c6b3ee09232a3b48f495 \ + --hash=sha256:a0ae6637b173d0c40b9c1462e12a7a2000a71a3258fa88756a34c7d38926911c \ + --hash=sha256:a31d21089894942f7d9a8df166b495101b7258ff11ae0abec58e32daf8088813 \ + --hash=sha256:a3442c31c11088e462d44a644a454d48110f0588de830921fd201060ff19612a \ + --hash=sha256:ab9524e45ee809a083338a749af3b53cc7efec458c3ad084361c1dbf7aaf82a2 \ + --hash=sha256:b1481c048fe787f65e34cb06f7d6824376d5d99f1231eae4778bbe5c3831076d \ + --hash=sha256:b8c837ab90c455f3ea8e68bee143472ee87828bff19ba19776e16ff961425b57 \ + --hash=sha256:bbf2c3f04ff50f16404ce70f822cdc59760e5e2d7965905f0e700270feb2bbfc \ + --hash=sha256:bbf9c2a589be7414ac4a534d54e4517d03f1cbb142c0041191b729c2fa23f320 \ + --hash=sha256:bcd5bf4132e6a8d3eb54b8d56885f3d3a38ecd7ecae8426ecf7d9673b270de43 \ + --hash=sha256:c14c16831b565707149c742d87a6203eb5597f4329278446d5c0ae7a1a43928e \ + --hash=sha256:c49f3e379177f4477f929097f7ed4b0622a586b0aa40c07ac8c0f8e40659a1ac \ + --hash=sha256:c92b89bffc660f1274779cb6fbb290ec1f90d6dfe14492523a0667f10170de26 \ + --hash=sha256:cd66152561632ed4b2a9192e7f8e5a1d41e28f58120b4761622e0355f0fe034c \ + --hash=sha256:cf1ad338620249f8dd6d4b6a91a69d1f265387df3697ad5dc996305cf6c26fb2 \ + --hash=sha256:d07b52c8c450f9366c34aa205754355e933922c79135125541daae6cbf31c799 \ + --hash=sha256:d0d12fe78dcf60efa205e9a63f395b5d343e801cf31e5e1dda0d2c1fb618073d \ + --hash=sha256:d4ee1d240b84e2f213565f0ec08caef27a0e657d4c42859809155cf3a29d1735 \ + --hash=sha256:d959fe96e5c2712c1876d69af0507d98f0b0e8d81bee14cfb3f6737470205419 \ + --hash=sha256:dcaef817e13eafa547cdfdc5284fe77970b891f731266545aae08d6cce52161e \ + --hash=sha256:df4e82e68f43a07735ae70a2d84c0353e58e20add20ec0af611f32cd5ba43fb4 \ + --hash=sha256:ec8cfe2295f3e5e44c51f57272afbd69414ae629ec7c6b27f5a410efc78b70a0 \ + --hash=sha256:ec9dd328016d8d25702a24ee274932aebf6be9787ed1c28d021945d264235b3c \ + --hash=sha256:ef9b85fa1bc91c4db24407e7c4da93a5822a73dd4513d67b454ca7064e8dc6a3 \ + --hash=sha256:f3bf60444269345d712838bb11cc4eadaf51ff1a364ae39ce87a5ca8ad3bb2c8 \ + --hash=sha256:f452cc1436151387d3d50533523291d5f77c6bc7913c116eb985304abdbd9ec9 \ + --hash=sha256:f7917697bcaa3bc3e83db91aa3a0e448bf5cde43c84b7fc1ae2427d2417c0224 \ + --hash=sha256:f90575e9fe3aae2c1e686393a9689c724cd00045275407f71771ae5d690ccf38 \ + --hash=sha256:fb382fd7b4377363cc9f13ba7c819c3c78ed97c36a82f16f3f92f108c787cbbf \ + --hash=sha256:fb9f59f3848edf186a76446eb8bcf4c900fe147cb756fbbd730ef43b2e67c6a7 \ + --hash=sha256:fc2931ac9ce9c61c9968989ec831d3a5e6fcaaff9474e7cfa8de80b7aff5a093 # via # -r requirements.in # aiohttp diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index 3d117d0f634e..b453bb1cb842 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -100,7 +100,6 @@ paths: # legacy core files which throw exceptions. We can add to this list but strongly prefer # StausOr where possible. - source/common/router/route_config_update_receiver_impl.cc - - source/common/listener_manager/listener_impl.cc - source/common/upstream/cluster_manager_impl.cc - source/common/upstream/upstream_impl.cc - source/common/network/listen_socket_impl.cc diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 452a48b9a23e..cd5d25bc3fbb 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -45,6 +45,7 @@ deadcode DFP Dynatrace DOM +DONTFRAG Gasd GiB IPTOS