From 93b7596b8baeac0d1bcf12e4761150c230a2707d Mon Sep 17 00:00:00 2001 From: Jose Ulises Nino Rivera Date: Thu, 29 Oct 2020 14:05:05 -0700 Subject: [PATCH 1/9] apple dns: resolve IP addresses without calling Apple APIs (#13698) Commit Message: apple dns - resolve IP addresses without calling Apple APIs Additional Description: After production testing with this resolver implementation it became empirically demonstrable that Apple's API issues queries to the DNS server for dns_name that might already be an IP address. In contrast, c-ares synchronously resolves such cases. Moreover, some DNS servers might not resolve IP addresses and thus result in callback targets that never get a resolution. Therefore, short circuiting was added to parse the dns_name into an internet address and synchronously issue the ResolveCb; only if the parsing throws an exception the resolver issues a call to Apple's API. Risk Level: med - important changes to the resolver, but the resolver has low production usage. Testing: local testing with iOS devices with Envoy Mobile. And added unit tests. Signed-off-by: Jose Nino --- source/common/network/apple_dns_impl.cc | 59 ++++++++++++++-------- source/common/network/utility.h | 1 + test/common/network/apple_dns_impl_test.cc | 48 +++++++++--------- 3 files changed, 63 insertions(+), 45 deletions(-) diff --git a/source/common/network/apple_dns_impl.cc b/source/common/network/apple_dns_impl.cc index 2755fa64ad28..dd2a75e4c0af 100644 --- a/source/common/network/apple_dns_impl.cc +++ b/source/common/network/apple_dns_impl.cc @@ -113,22 +113,44 @@ ActiveDnsQuery* AppleDnsResolverImpl::resolve(const std::string& dns_name, DnsLookupFamily dns_lookup_family, ResolveCb callback) { ENVOY_LOG(debug, "DNS resolver resolve={}", dns_name); - std::unique_ptr pending_resolution( - new PendingResolution(*this, callback, dispatcher_, main_sd_ref_, dns_name)); - DNSServiceErrorType error = pending_resolution->dnsServiceGetAddrInfo(dns_lookup_family); - if (error != kDNSServiceErr_NoError) { - ENVOY_LOG(warn, "DNS resolver error ({}) in dnsServiceGetAddrInfo for {}", error, dns_name); - return nullptr; - } + Address::InstanceConstSharedPtr address{}; + try { + // When an IP address is submitted to c-ares in DnsResolverImpl, c-ares synchronously returns + // the IP without submitting a DNS query. Because Envoy has come to rely on this behavior, this + // resolver implements a similar resolution path to avoid making improper DNS queries for + // resolved IPs. + address = Utility::parseInternetAddress(dns_name); + ENVOY_LOG(debug, "DNS resolver resolved ({}) to ({}) without issuing call to Apple API", + dns_name, address->asString()); + } catch (const EnvoyException& e) { + // Resolution via Apple APIs + ENVOY_LOG(debug, "DNS resolver local resolution failed with: {}", e.what()); + std::unique_ptr pending_resolution( + new PendingResolution(*this, callback, dispatcher_, main_sd_ref_, dns_name)); + + DNSServiceErrorType error = pending_resolution->dnsServiceGetAddrInfo(dns_lookup_family); + if (error != kDNSServiceErr_NoError) { + ENVOY_LOG(warn, "DNS resolver error ({}) in dnsServiceGetAddrInfo for {}", error, dns_name); + return nullptr; + } - // If the query was synchronously resolved, there is no need to return the query. - if (pending_resolution->synchronously_completed_) { - return nullptr; + // If the query was synchronously resolved in the Apple API call, there is no need to return the + // query. + if (pending_resolution->synchronously_completed_) { + return nullptr; + } + + pending_resolution->owned_ = true; + return pending_resolution.release(); } - pending_resolution->owned_ = true; - return pending_resolution.release(); + ASSERT(address != nullptr); + // Finish local, synchronous resolution. This needs to happen outside of the exception block above + // as the callback itself can throw. + callback(DnsResolver::ResolutionStatus::Success, + {DnsResponse(address, std::chrono::seconds(60))}); + return nullptr; } void AppleDnsResolverImpl::addPendingQuery(PendingResolution* query) { @@ -146,16 +168,9 @@ void AppleDnsResolverImpl::flushPendingQueries(const bool with_error) { for (std::set::iterator it = queries_with_pending_cb_.begin(); it != queries_with_pending_cb_.end(); ++it) { auto query = *it; - try { - ASSERT(query->pending_cb_); - query->callback_(query->pending_cb_->status_, std::move(query->pending_cb_->responses_)); - } catch (const std::exception& e) { - ENVOY_LOG(warn, "std::exception in DNSService callback: {}", e.what()); - throw EnvoyException(e.what()); - } catch (...) { - ENVOY_LOG(warn, "Unknown exception in DNSService callback"); - throw EnvoyException("unknown"); - } + + ASSERT(query->pending_cb_); + query->callback_(query->pending_cb_->status_, std::move(query->pending_cb_->responses_)); if (query->owned_) { ENVOY_LOG(debug, "Resolution for {} completed (async)", query->dns_name_); diff --git a/source/common/network/utility.h b/source/common/network/utility.h index be64071e9ea6..606baf8bd5b1 100644 --- a/source/common/network/utility.h +++ b/source/common/network/utility.h @@ -72,6 +72,7 @@ class Utility { * Resolve a URL. * @param url supplies the url to resolve. * @return Address::InstanceConstSharedPtr the resolved address. + * @throw EnvoyException if url is invalid. */ static Address::InstanceConstSharedPtr resolveUrl(const std::string& url); diff --git a/test/common/network/apple_dns_impl_test.cc b/test/common/network/apple_dns_impl_test.cc index f9bffcc31937..acaed880d758 100644 --- a/test/common/network/apple_dns_impl_test.cc +++ b/test/common/network/apple_dns_impl_test.cc @@ -97,16 +97,15 @@ class AppleDnsImplTest : public testing::Test { }); } - template ActiveDnsQuery* resolveWithException(const std::string& address, - const DnsLookupFamily lookup_family, T exception_object) { - return resolver_->resolve(address, lookup_family, - [exception_object](DnsResolver::ResolutionStatus status, - std::list&& results) -> void { - UNREFERENCED_PARAMETER(status); - UNREFERENCED_PARAMETER(results); - throw exception_object; - }); + const DnsLookupFamily lookup_family) { + return resolver_->resolve( + address, lookup_family, + [](DnsResolver::ResolutionStatus status, std::list&& results) -> void { + UNREFERENCED_PARAMETER(status); + UNREFERENCED_PARAMETER(results); + throw EnvoyException("Envoy exception"); + }); } protected: @@ -154,24 +153,14 @@ TEST_F(AppleDnsImplTest, DnsIpAddressVersion) { } TEST_F(AppleDnsImplTest, CallbackException) { - EXPECT_NE(nullptr, resolveWithException("1.2.3.4", DnsLookupFamily::V4Only, - EnvoyException("Envoy exception"))); + EXPECT_NE(nullptr, resolveWithException("google.com", DnsLookupFamily::V4Only)); EXPECT_THROW_WITH_MESSAGE(dispatcher_->run(Event::Dispatcher::RunType::Block), EnvoyException, "Envoy exception"); } -TEST_F(AppleDnsImplTest, CallbackException2) { - EXPECT_NE(nullptr, resolveWithException("1.2.3.4", DnsLookupFamily::V4Only, - std::runtime_error("runtime error"))); - EXPECT_THROW_WITH_MESSAGE(dispatcher_->run(Event::Dispatcher::RunType::Block), EnvoyException, - "runtime error"); -} - -TEST_F(AppleDnsImplTest, CallbackException3) { - EXPECT_NE(nullptr, - resolveWithException("1.2.3.4", DnsLookupFamily::V4Only, std::string())); - EXPECT_THROW_WITH_MESSAGE(dispatcher_->run(Event::Dispatcher::RunType::Block), EnvoyException, - "unknown"); +TEST_F(AppleDnsImplTest, CallbackExceptionLocalResolution) { + EXPECT_THROW_WITH_MESSAGE(resolveWithException("1.2.3.4", DnsLookupFamily::V4Only), + EnvoyException, "Envoy exception"); } // Validate working of cancellation provided by ActiveDnsQuery return. @@ -194,6 +183,19 @@ TEST_F(AppleDnsImplTest, Timeout) { dispatcher_->run(Event::Dispatcher::RunType::Block); } +TEST_F(AppleDnsImplTest, LocalResolution) { + auto pending_resolution = resolver_->resolve( + "0.0.0.0", DnsLookupFamily::Auto, + [](DnsResolver::ResolutionStatus status, std::list&& results) -> void { + EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(1, results.size()); + EXPECT_EQ("0.0.0.0:0", results.front().address_->asString()); + EXPECT_EQ(std::chrono::seconds(60), results.front().ttl_); + }); + EXPECT_EQ(nullptr, pending_resolution); + // Note that the dispatcher does NOT have to run because resolution is synchronous. +} + // This class compliments the tests above by using a mocked Apple API that allows finer control over // error conditions, and callback firing. class AppleDnsImplFakeApiTest : public testing::Test { From 0bcd984e09cb3275d1a085df38a47c37160ca691 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 29 Oct 2020 21:51:51 +0000 Subject: [PATCH 2/9] examples: Add dynamic configuration (filesystem) sandbox (#13783) Signed-off-by: Ryan Northey --- ...sponse-config-active-clusters-updated.json | 31 +++++ .../response-config-active-clusters.json | 31 +++++ .../dynamic-configuration-filesystem.rst | 122 ++++++++++++++++++ docs/root/start/sandboxes/index.rst | 1 + examples/BUILD | 1 + examples/dynamic-config-fs/Dockerfile-proxy | 5 + examples/dynamic-config-fs/README.md | 2 + examples/dynamic-config-fs/configs/cds.yaml | 15 +++ examples/dynamic-config-fs/configs/lds.yaml | 26 ++++ .../dynamic-config-fs/docker-compose.yaml | 23 ++++ examples/dynamic-config-fs/envoy.yaml | 16 +++ examples/dynamic-config-fs/verify.sh | 32 +++++ 12 files changed, 305 insertions(+) create mode 100644 docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters-updated.json create mode 100644 docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters.json create mode 100644 docs/root/start/sandboxes/dynamic-configuration-filesystem.rst create mode 100644 examples/dynamic-config-fs/Dockerfile-proxy create mode 100644 examples/dynamic-config-fs/README.md create mode 100644 examples/dynamic-config-fs/configs/cds.yaml create mode 100644 examples/dynamic-config-fs/configs/lds.yaml create mode 100644 examples/dynamic-config-fs/docker-compose.yaml create mode 100644 examples/dynamic-config-fs/envoy.yaml create mode 100755 examples/dynamic-config-fs/verify.sh diff --git a/docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters-updated.json b/docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters-updated.json new file mode 100644 index 000000000000..43b676e72d66 --- /dev/null +++ b/docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters-updated.json @@ -0,0 +1,31 @@ +[ + { + "cluster": { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "example_proxy_cluster", + "type": "LOGICAL_DNS", + "connect_timeout": "5s", + "dns_lookup_family": "V4_ONLY", + "load_assignment": { + "cluster_name": "example_proxy_cluster", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "service2", + "port_value": 8080 + } + } + } + } + ] + } + ] + } + }, + "last_updated": "2020-10-25T20:37:05.838Z" + } +] diff --git a/docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters.json b/docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters.json new file mode 100644 index 000000000000..3813a9a9c56c --- /dev/null +++ b/docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters.json @@ -0,0 +1,31 @@ +[ + { + "cluster": { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "example_proxy_cluster", + "type": "LOGICAL_DNS", + "connect_timeout": "5s", + "dns_lookup_family": "V4_ONLY", + "load_assignment": { + "cluster_name": "example_proxy_cluster", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "service1", + "port_value": 8080 + } + } + } + } + ] + } + ] + } + }, + "last_updated": "2020-10-25T20:37:05.838Z" + } +] diff --git a/docs/root/start/sandboxes/dynamic-configuration-filesystem.rst b/docs/root/start/sandboxes/dynamic-configuration-filesystem.rst new file mode 100644 index 000000000000..d71bee0e9673 --- /dev/null +++ b/docs/root/start/sandboxes/dynamic-configuration-filesystem.rst @@ -0,0 +1,122 @@ +.. _install_sandboxes_dynamic_config_fs: + +Dynamic configuration (filesystem) +================================== + +This example walks through configuring Envoy using filesystem-based dynamic configuration. + +It demonstrates how configuration provided to Envoy dynamically can be updated without +restarting the server. + +.. include:: _include/docker-env-setup.rst + +Change directory to ``examples/dynamic-config-fs`` in the Envoy repository. + +Step 3: Start the proxy container +********************************* + +.. note:: + + If you are running on a system with strict ``umask`` you will need to ``chmod`` the dynamic config + files which are mounted into the container: + + .. code-block:: console + + $ umask + 027 + $ pwd + envoy/examples/dynamic-config-fs + $ chmod go+r configs/* + $ chmod go+x configs + +Build and start the containers. + +This should also start two upstream ``HTTP`` echo servers, ``service1`` and ``service2``. + +.. code-block:: console + + $ pwd + envoy/examples/dynamic-config-fs + $ docker-compose build --pull + $ docker-compose up -d + $ docker-compose ps + + Name Command State Ports + ------------------------------------------------------------------------------------------------------------------------ + dynamic-config-fs_proxy_1 /docker-entrypoint.sh /usr ... Up 0.0.0.0:10000->10000/tcp, 0.0.0.0:19000->19000/tcp + dynamic-config-fs_service1_1 /bin/echo-server Up 8080/tcp + dynamic-config-fs_service2_1 /bin/echo-server Up 8080/tcp + +Step 4: Check web response +************************** + +You should be able to make a request to port ``10000``, which will be served by ``service1``. + +.. code-block:: console + + $ curl -s http://localhost:10000 + Request served by service1 + + HTTP/2.0 GET / + + Host: localhost:10000 + User-Agent: curl/7.72.0 + Accept: */* + X-Forwarded-Proto: http + X-Request-Id: 6672902d-56ca-456c-be6a-992a603cab9a + X-Envoy-Expected-Rq-Timeout-Ms: 15000 + +Step 5: Dump Envoy's ``dynamic_active_clusters`` config +******************************************************* + +If you now dump the proxy’s ``dynamic_active_clusters`` configuration, you should see it is configured with +the ``example_proxy_cluster`` pointing to ``service1``. + +.. code-block:: console + + $ curl -s http://localhost:19000/config_dump | jq -r '.configs[1].dynamic_active_clusters' + +.. literalinclude:: _include/dynamic-config-fs/response-config-active-clusters.json + :language: json + :emphasize-lines: 10, 18-19 + +Step 5: Edit ``configs/cds.yaml`` file to update upstream cluster +***************************************************************** + +The example setup provides two dynamic configuration files: + +- :download:`configs/cds.yaml <_include/dynamic-config-fs/configs/cds.yaml>` to provide a :ref:`Cluster + discovery service (CDS) `. +- :download:`configs/lds.yaml <_include/dynamic-config-fs/configs/lds.yaml>` to provide a :ref:`Listener + discovery service (CDS) `. + +Edit ``configs/cds.yaml`` in the dynamic configuration example folder and change the cluster address +from ``service1`` to ``service2``: + +.. literalinclude:: _include/dynamic-config-fs/configs/cds.yaml + :language: yaml + :linenos: + :lines: 7-15 + :lineno-start: 7 + :emphasize-lines: 8 + +Step 6: Check Envoy uses updated configuration +********************************************** + +Checking the web response again, the request should now be handled by ``service2``: + +.. code-block:: console + + $ curl http://localhost:10000 | grep "served by" + Request served by service2 + +Dumping the ``dynamic_active_clusters``, the ``example_proxy_cluster`` should now be +configured to proxy to ``service2``: + +.. code-block:: console + + $ curl -s http://localhost:19000/config_dump jq -r '.configs[1].dynamic_active_clusters' + +.. literalinclude:: _include/dynamic-config-fs/response-config-active-clusters-updated.json + :language: json + :emphasize-lines: 10, 18-19 diff --git a/docs/root/start/sandboxes/index.rst b/docs/root/start/sandboxes/index.rst index 550776cd2503..88db6644e856 100644 --- a/docs/root/start/sandboxes/index.rst +++ b/docs/root/start/sandboxes/index.rst @@ -14,6 +14,7 @@ features. The following sandboxes are available: cache cors csrf + dynamic-configuration-filesystem dynamic-configuration-control-plane ext_authz fault_injection diff --git a/examples/BUILD b/examples/BUILD index f5fdff5710a4..59d945583b06 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -18,6 +18,7 @@ filegroup( ], exclude = [ "cache/responses.yaml", + "dynamic-config-fs/**/*", "jaeger-native-tracing/*", "**/*docker-compose*.yaml", ], diff --git a/examples/dynamic-config-fs/Dockerfile-proxy b/examples/dynamic-config-fs/Dockerfile-proxy new file mode 100644 index 000000000000..f70f44311461 --- /dev/null +++ b/examples/dynamic-config-fs/Dockerfile-proxy @@ -0,0 +1,5 @@ +FROM envoyproxy/envoy-dev:latest + +COPY ./envoy.yaml /etc/envoy.yaml +RUN chmod go+r /etc/envoy.yaml +CMD ["/usr/local/bin/envoy", "-c /etc/envoy.yaml", "-l", "debug"] diff --git a/examples/dynamic-config-fs/README.md b/examples/dynamic-config-fs/README.md new file mode 100644 index 000000000000..3cb1ed49d940 --- /dev/null +++ b/examples/dynamic-config-fs/README.md @@ -0,0 +1,2 @@ +To learn about this sandbox and for instructions on how to run it please head over +to the [Envoy docs](https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/dynamic-configuration-filesystem.html). diff --git a/examples/dynamic-config-fs/configs/cds.yaml b/examples/dynamic-config-fs/configs/cds.yaml new file mode 100644 index 000000000000..3f661da7d7ab --- /dev/null +++ b/examples/dynamic-config-fs/configs/cds.yaml @@ -0,0 +1,15 @@ +resources: +- "@type": type.googleapis.com/envoy.config.cluster.v3.Cluster + name: example_proxy_cluster + connect_timeout: 1s + type: strict_dns + http2_protocol_options: {} + load_assignment: + cluster_name: example_proxy_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: service1 + port_value: 8080 diff --git a/examples/dynamic-config-fs/configs/lds.yaml b/examples/dynamic-config-fs/configs/lds.yaml new file mode 100644 index 000000000000..4770f538ac25 --- /dev/null +++ b/examples/dynamic-config-fs/configs/lds.yaml @@ -0,0 +1,26 @@ +resources: +- "@type": type.googleapis.com/envoy.config.listener.v3.Listener + name: listener_0 + address: + socket_address: + address: 0.0.0.0 + port_value: 10000 + filter_chains: + - filters: + name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + http_filters: + - name: envoy.router + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: example_proxy_cluster diff --git a/examples/dynamic-config-fs/docker-compose.yaml b/examples/dynamic-config-fs/docker-compose.yaml new file mode 100644 index 000000000000..b3aed8c7e8ad --- /dev/null +++ b/examples/dynamic-config-fs/docker-compose.yaml @@ -0,0 +1,23 @@ +version: "3.7" +services: + + proxy: + build: + context: . + dockerfile: Dockerfile-proxy + depends_on: + - service1 + - service2 + ports: + - 10000:10000 + - 19000:19000 + volumes: + - ./configs:/var/lib/envoy + + service1: + image: jmalloc/echo-server + hostname: service1 + + service2: + image: jmalloc/echo-server + hostname: service2 diff --git a/examples/dynamic-config-fs/envoy.yaml b/examples/dynamic-config-fs/envoy.yaml new file mode 100644 index 000000000000..27587e0dbca4 --- /dev/null +++ b/examples/dynamic-config-fs/envoy.yaml @@ -0,0 +1,16 @@ +node: + id: id_1 + cluster: test + +dynamic_resources: + cds_config: + path: /var/lib/envoy/cds.yaml + lds_config: + path: /var/lib/envoy/lds.yaml + +admin: + access_log_path: "/dev/null" + address: + socket_address: + address: 0.0.0.0 + port_value: 19000 diff --git a/examples/dynamic-config-fs/verify.sh b/examples/dynamic-config-fs/verify.sh new file mode 100755 index 000000000000..1799bd9575f9 --- /dev/null +++ b/examples/dynamic-config-fs/verify.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +export NAME=dynamic-config-fs + +chmod go+r configs/* +chmod go+rx configs + +# shellcheck source=examples/verify-common.sh +. "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh" + +run_log "Check for response comes from service1 upstream" +responds_with \ + "Request served by service1" \ + http://localhost:10000 + +run_log "Check config for active clusters pointing to service1" +curl -s http://localhost:19000/config_dump \ + | jq -r '.configs[1].dynamic_active_clusters' \ + | grep '"address": "service1"' + +run_log "Set upstream to service2" +sed -i s/service1/service2/ configs/cds.yaml + +run_log "Check for response comes from service2 upstream" +responds_with \ + "Request served by service2" \ + http://localhost:10000 + +run_log "Check config for active clusters pointing to service2" +curl -s http://localhost:19000/config_dump \ + | jq -r '.configs[1].dynamic_active_clusters' \ + | grep '"address": "service2"' From 51573d32455381cec164bbf58af06ea21cc2abac Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Thu, 29 Oct 2020 15:52:01 -0700 Subject: [PATCH 3/9] wasm: strip only Custom Sections with precompiled Wasm modules. (#13775) Signed-off-by: Piotr Sikora --- bazel/repository_locations.bzl | 6 +++--- test/tools/wee8_compile/wee8_compile.cc | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 677bfb4e9ecd..c9a3dbd343c9 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -830,8 +830,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "WebAssembly for Proxies (C++ host implementation)", project_desc = "WebAssembly for Proxies (C++ host implementation)", project_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-host", - version = "d54b3795e7c3e61015dac2c2110b0da2be999b8e", - sha256 = "e95ad57b6b550b039d4baa35c896f7f523e427b7278ba56f22fac6e1bef8c7f0", + version = "40fd3d03842c07d65fed907a6b6ed0f89d68d531", + sha256 = "b5ae746e66b6209ea0cce86d6c21de99dacbec1da9cdadd53a9ec46bc296a3ba", strip_prefix = "proxy-wasm-cpp-host-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-host/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -842,7 +842,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - release_date = "2020-10-22", + release_date = "2020-10-27", cpe = "N/A", ), emscripten_toolchain = dict( diff --git a/test/tools/wee8_compile/wee8_compile.cc b/test/tools/wee8_compile/wee8_compile.cc index 42cbfea08a18..a1b2906ab281 100644 --- a/test/tools/wee8_compile/wee8_compile.cc +++ b/test/tools/wee8_compile/wee8_compile.cc @@ -126,10 +126,22 @@ wasm::vec stripWasmModule(const wasm::vec& module) { std::cerr << "ERROR: Failed to parse corrupted Wasm module." << std::endl; return wasm::vec::invalid(); } - if (section_type != 0 /* custom section */) { - stripped.insert(stripped.end(), section_start, pos + section_len); + if (section_type == 0 /* custom section */) { + const auto section_data_start = pos; + const auto section_name_len = parseVarint(pos, end); + if (section_name_len == static_cast(-1) || pos + section_name_len > end) { + std::cerr << "ERROR: Failed to parse corrupted Wasm module." << std::endl; + return wasm::vec::invalid(); + } + auto section_name = std::string(pos, section_name_len); + if (section_name.find("precompiled_") == std::string::npos) { + stripped.insert(stripped.end(), section_start, section_data_start + section_len); + } + pos = section_data_start + section_len; + } else { + pos += section_len; + stripped.insert(stripped.end(), section_start, pos /* section end */); } - pos += section_len; } return wasm::vec::make(stripped.size(), stripped.data()); From af418e1096a386000f936744a1a884b6ce87cee0 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Fri, 30 Oct 2020 12:12:06 +0530 Subject: [PATCH 4/9] Removed Circle-CI reference. (#13824) Signed-off-by: Manish Kumar --- CONTRIBUTING.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76081a32d440..91642701329d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -345,14 +345,6 @@ should only be done to correct a DCO mistake. ## Triggering CI re-run without making changes -To rerun failed tasks in Circle-CI, add a comment with the line - -``` -/retest-circle -``` - -in it. This should rebuild only the failed tasks. - To rerun failed tasks in Azure pipelines, add a comment with the line ``` From f7f870917bd25dc8200bcce8c259bd588cafb559 Mon Sep 17 00:00:00 2001 From: Zach Reyes <39203661+zasweq@users.noreply.github.com> Date: Fri, 30 Oct 2020 09:48:25 -0400 Subject: [PATCH 5/9] [fuzz] Scaled Load balancer fuzz to 60k hosts (#13771) * Scaled load balancer fuzzer up to 10k hosts Signed-off-by: Zach --- test/common/upstream/load_balancer_fuzz.proto | 18 ++-- .../upstream/load_balancer_fuzz_base.cc | 86 +++++++++++++------ .../common/upstream/load_balancer_fuzz_base.h | 17 ++-- .../random_256_ports | 6 +- .../random_NoHosts | 10 ++- .../random_load_balancer_corpus/random_Normal | 15 +++- ...h-55abbf82c64b5a62e299b93d7b254045471199c9 | 41 +++++++++ ...h-5a64c214e5c2038299c8f2e85f143f1163077c1b | 25 ++++++ ...h-ba5efdfd9c412a8507087120783fe6529b1ac0cb | 4 +- .../random_largest-port-value | 4 +- .../random_many_choose_hosts | 8 +- .../random_max_ports | 15 +++- .../random_overflowing_ports | 15 +++- ...t-eed4596101efb3e737f736c8d5bcd4f0815a8728 | 20 ++++- .../random_slow-unit-test | 20 ++++- .../random_test_something | 15 +++- ...t-6b0d6b83136a4cf0b9ccd468f11207a792859d43 | 10 ++- ...t-9144cfbb40b5101ecc28b205b10e6c36a72aae83 | 10 ++- .../random_with-locality | 15 +++- .../random_with-locality-high-number-of-hosts | 58 +++++++++++++ .../random_with_locality-50000-hosts | 58 +++++++++++++ test/fuzz/random.h | 24 +++--- test/fuzz/random_test.cc | 20 +++-- 23 files changed, 419 insertions(+), 95 deletions(-) create mode 100644 test/common/upstream/random_load_balancer_corpus/random_crash-55abbf82c64b5a62e299b93d7b254045471199c9 create mode 100644 test/common/upstream/random_load_balancer_corpus/random_crash-5a64c214e5c2038299c8f2e85f143f1163077c1b create mode 100644 test/common/upstream/random_load_balancer_corpus/random_with-locality-high-number-of-hosts create mode 100644 test/common/upstream/random_load_balancer_corpus/random_with_locality-50000-hosts diff --git a/test/common/upstream/load_balancer_fuzz.proto b/test/common/upstream/load_balancer_fuzz.proto index c4b1ead2c7d0..c152adc21248 100644 --- a/test/common/upstream/load_balancer_fuzz.proto +++ b/test/common/upstream/load_balancer_fuzz.proto @@ -8,14 +8,17 @@ import "google/protobuf/empty.proto"; message UpdateHealthFlags { // The host priority determines what host set within the priority set which will get updated. - uint64 host_priority = 1; + uint32 host_priority = 1; // These will determine how many hosts will get placed into health hosts, degraded hosts, and // excluded hosts from the full host list. uint32 num_healthy_hosts = 2; uint32 num_degraded_hosts = 3; uint32 num_excluded_hosts = 4; // This is used to determine which hosts get marked as healthy, degraded, and excluded. - bytes random_bytestring = 5 [(validate.rules).bytes = {min_len: 1, max_len: 256}]; + // TODO: What should this be capped at? There might be some efficiency/code coverage trade off + // dependent on the amount of digits this random_bytestring is allowed to scale too. + repeated uint32 random_bytestring = 5 + [(validate.rules).repeated = {min_items: 1, max_items: 60000}]; } message LbAction { @@ -32,13 +35,14 @@ message LbAction { } message SetupPriorityLevel { - uint32 num_hosts_in_priority_level = 1 [(validate.rules).uint32.lte = 500]; - uint32 num_hosts_locality_a = 2 [(validate.rules).uint32.lte = 500]; - uint32 num_hosts_locality_b = 3 [(validate.rules).uint32.lte = 500]; + uint32 num_hosts_in_priority_level = 1 [(validate.rules).uint32.lte = 60000]; + uint32 num_hosts_locality_a = 2 [(validate.rules).uint32.lte = 60000]; + uint32 num_hosts_locality_b = 3 [(validate.rules).uint32.lte = 60000]; // Hard cap at 3 localities for simplicity - uint32 num_hosts_locality_c = 4 [(validate.rules).uint32.lte = 500]; + uint32 num_hosts_locality_c = 4 [(validate.rules).uint32.lte = 60000]; // For choosing which hosts go in which locality - bytes random_bytestring = 5 [(validate.rules).bytes = {min_len: 1, max_len: 256}]; + repeated uint32 random_bytestring = 5 + [(validate.rules).repeated = {min_items: 1, max_items: 60000}]; } // This message represents what LoadBalancerFuzzBase will interact with, performing setup of host sets and calling into load balancers. diff --git a/test/common/upstream/load_balancer_fuzz_base.cc b/test/common/upstream/load_balancer_fuzz_base.cc index 6741f95f7581..1bf578b214df 100644 --- a/test/common/upstream/load_balancer_fuzz_base.cc +++ b/test/common/upstream/load_balancer_fuzz_base.cc @@ -6,14 +6,28 @@ namespace Envoy { namespace Upstream { namespace { -// TODO(zasweq): This will be relaxed in the future in order to fully represent the state space -// possible within Load Balancing. In it's current state, it is too slow (particularly due to calls -// to makeTestHost()) to scale up hosts. Once this is made more efficient, this number will be -// increased. -constexpr uint32_t MaxNumHostsPerPriorityLevel = 256; + +constexpr uint32_t MaxNumHostsPerPriorityLevel = 60000; + +// Helper function for converting repeated proto fields to byte vectors to pass into random subset +std::vector +constructByteVectorForRandom(const Protobuf::RepeatedField& random_bytestring) { + std::vector random_bytestring_vector(random_bytestring.begin(), + random_bytestring.end()); + return random_bytestring_vector; +} } // namespace +HostVector +LoadBalancerFuzzBase::initializeHostsForUseInFuzzing(std::shared_ptr info) { + HostVector hosts; + for (uint32_t i = 1; i <= 60000; ++i) { + hosts.push_back(makeTestHost(info, "tcp://127.0.0.1:" + std::to_string(i))); + } + return hosts; +} + void LoadBalancerFuzzBase::initializeASingleHostSet( const test::common::upstream::SetupPriorityLevel& setup_priority_level, const uint8_t priority_level, uint16_t& port) { @@ -22,17 +36,19 @@ void LoadBalancerFuzzBase::initializeASingleHostSet( priority_level, num_hosts_in_priority_level); MockHostSet& host_set = *priority_set_.getMockHostSet(priority_level); uint32_t hosts_made = 0; - // Cap each host set at 256 hosts for efficiency - Leave port clause in for future changes + // Cap each host set at 60000 hosts - however, let port number enforce a max of 60k hosts across + // all priority levels. while (hosts_made < std::min(num_hosts_in_priority_level, MaxNumHostsPerPriorityLevel) && - port < 65535) { - host_set.hosts_.push_back(makeTestHost(info_, "tcp://127.0.0.1:" + std::to_string(port))); + port < 60000) { + host_set.hosts_.push_back(initialized_hosts_[port]); ++port; ++hosts_made; } - Fuzz::ProperSubsetSelector subset_selector(setup_priority_level.random_bytestring()); + Fuzz::ProperSubsetSelector subset_selector( + constructByteVectorForRandom(setup_priority_level.random_bytestring())); - const std::vector> localities = subset_selector.constructSubsets( + const std::vector> localities = subset_selector.constructSubsets( {setup_priority_level.num_hosts_locality_a(), setup_priority_level.num_hosts_locality_b(), setup_priority_level.num_hosts_locality_c()}, host_set.hosts_.size()); @@ -44,7 +60,7 @@ void LoadBalancerFuzzBase::initializeASingleHostSet( std::array locality_indexes = {locality_a, locality_b, locality_c}; for (uint8_t locality = 0; locality < locality_indexes.size(); locality++) { - for (uint8_t index : localities[locality]) { + for (uint32_t index : localities[locality]) { locality_indexes[locality].push_back(host_set.hosts_[index]); locality_indexes_[index] = locality; } @@ -58,8 +74,16 @@ void LoadBalancerFuzzBase::initializeASingleHostSet( // Initializes random and fixed host sets void LoadBalancerFuzzBase::initializeLbComponents( const test::common::upstream::LoadBalancerTestCase& input) { + static NiceMock info; + static std::shared_ptr info_pointer{std::shared_ptr{}, &info}; + + // Will statically initialize 60000 hosts in this vector, so each fuzz run doesn't construct new + // hosts to use. This will require clearing of state after each run. + static HostVector initialized_hosts = initializeHostsForUseInFuzzing(info_pointer); + initialized_hosts_ = initialized_hosts; + random_.initializeSeed(input.seed_for_prng()); - uint16_t port = 80; + uint16_t port = 1; for (uint8_t priority_of_host_set = 0; priority_of_host_set < input.setup_priority_levels().size(); ++priority_of_host_set) { initializeASingleHostSet(input.setup_priority_levels().at(priority_of_host_set), @@ -71,11 +95,10 @@ void LoadBalancerFuzzBase::initializeLbComponents( // Updating host sets is shared amongst all the load balancer tests. Since logically, we're just // setting the mock priority set to have certain values, and all load balancers interface with host // sets and their health statuses, this action maps to all load balancers. -void LoadBalancerFuzzBase::updateHealthFlagsForAHostSet(const uint64_t host_priority, - const uint32_t num_healthy_hosts, - const uint32_t num_degraded_hosts, - const uint32_t num_excluded_hosts, - const std::string random_bytestring) { +void LoadBalancerFuzzBase::updateHealthFlagsForAHostSet( + const uint64_t host_priority, const uint32_t num_healthy_hosts, + const uint32_t num_degraded_hosts, const uint32_t num_excluded_hosts, + const Protobuf::RepeatedField& random_bytestring) { const uint8_t priority_of_host_set = host_priority % num_priority_levels_; ENVOY_LOG_MISC(trace, "Updating health flags for host set at priority: {}", priority_of_host_set); MockHostSet& host_set = *priority_set_.getMockHostSet(priority_of_host_set); @@ -99,13 +122,13 @@ void LoadBalancerFuzzBase::updateHealthFlagsForAHostSet(const uint64_t host_prio EXCLUDED = 2, }; - Fuzz::ProperSubsetSelector subset_selector(random_bytestring); + Fuzz::ProperSubsetSelector subset_selector(constructByteVectorForRandom(random_bytestring)); - const std::vector> subsets = subset_selector.constructSubsets( + const std::vector> subsets = subset_selector.constructSubsets( {num_healthy_hosts, num_degraded_hosts, num_excluded_hosts}, host_set_size); // Healthy hosts are first subset - for (uint8_t index : subsets.at(HealthStatus::HEALTHY)) { + for (uint32_t index : subsets.at(HealthStatus::HEALTHY)) { host_set.healthy_hosts_.push_back(host_set.hosts_[index]); // No health flags for healthy } @@ -113,7 +136,7 @@ void LoadBalancerFuzzBase::updateHealthFlagsForAHostSet(const uint64_t host_prio absl::StrJoin(subsets.at(HealthStatus::HEALTHY), " ")); // Degraded hosts are second subset - for (uint8_t index : subsets.at(HealthStatus::DEGRADED)) { + for (uint32_t index : subsets.at(HealthStatus::DEGRADED)) { host_set.degraded_hosts_.push_back(host_set.hosts_[index]); // Health flags are not currently directly used by most load balancers, but // they may be added and also are used by other components. @@ -125,7 +148,7 @@ void LoadBalancerFuzzBase::updateHealthFlagsForAHostSet(const uint64_t host_prio absl::StrJoin(subsets.at(HealthStatus::DEGRADED), " ")); // Excluded hosts are third subset - for (uint8_t index : subsets.at(HealthStatus::EXCLUDED)) { + for (uint32_t index : subsets.at(HealthStatus::EXCLUDED)) { host_set.excluded_hosts_.push_back(host_set.hosts_[index]); // Health flags are not currently directly used by most load balancers, but // they may be added and also are used by other components. @@ -156,8 +179,8 @@ void LoadBalancerFuzzBase::updateHealthFlagsForAHostSet(const uint64_t host_prio // Iterate through subsets for (uint8_t health_status = 0; health_status < locality_health_statuses.size(); health_status++) { - for (uint8_t index : subsets.at(health_status)) { // Each subset logically represents a health - // status + for (uint32_t index : subsets.at(health_status)) { // Each subset logically represents a health + // status // If the host is in a locality, we have to update the corresponding health status host vector if (!(locality_indexes_.find(index) == locality_indexes_.end())) { // After computing the host index subsets, we want to propagate these changes to a host set @@ -225,5 +248,20 @@ void LoadBalancerFuzzBase::replay( } } +void LoadBalancerFuzzBase::clearStaticHostsHealthFlags() { + // The only outstanding health flags set are those that are set from hosts being placed in + // degraded and excluded. Thus, use the priority set pointer to know which flags to clear. + for (uint32_t priority_level = 0; priority_level < priority_set_.hostSetsPerPriority().size(); + ++priority_level) { + MockHostSet& host_set = *priority_set_.getMockHostSet(priority_level); + for (auto& host : host_set.degraded_hosts_) { + host->healthFlagClear(Host::HealthFlag::DEGRADED_ACTIVE_HC); + } + for (auto& host : host_set.excluded_hosts_) { + host->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); + } + } +} + } // namespace Upstream } // namespace Envoy diff --git a/test/common/upstream/load_balancer_fuzz_base.h b/test/common/upstream/load_balancer_fuzz_base.h index deeb4c82c216..fc9abcffbf0f 100644 --- a/test/common/upstream/load_balancer_fuzz_base.h +++ b/test/common/upstream/load_balancer_fuzz_base.h @@ -23,10 +23,10 @@ class LoadBalancerFuzzBase { // Initializes load balancer components shared amongst every load balancer, random_, and // priority_set_ void initializeLbComponents(const test::common::upstream::LoadBalancerTestCase& input); - void updateHealthFlagsForAHostSet(const uint64_t host_priority, const uint32_t num_healthy_hosts, - const uint32_t num_degraded_hosts, - const uint32_t num_excluded_hosts, - const std::string random_bytestring); + void + updateHealthFlagsForAHostSet(const uint64_t host_priority, const uint32_t num_healthy_hosts, + const uint32_t num_degraded_hosts, const uint32_t num_excluded_hosts, + const Protobuf::RepeatedField& random_bytestring); // These two actions have a lot of logic attached to them. However, all the logic that the load // balancer needs to run its algorithm is already encapsulated within the load balancer. Thus, // once the load balancer is constructed, all this class has to do is call lb_->peekAnotherHost() @@ -36,6 +36,8 @@ class LoadBalancerFuzzBase { ~LoadBalancerFuzzBase() = default; void replay(const Protobuf::RepeatedPtrField& actions); + void clearStaticHostsHealthFlags(); + // These public objects shared amongst all types of load balancers will be used to construct load // balancers in specific load balancer fuzz classes Stats::IsolatedStoreImpl stats_store_; @@ -43,7 +45,6 @@ class LoadBalancerFuzzBase { NiceMock runtime_; Random::PsuedoRandomGenerator64 random_; NiceMock priority_set_; - std::shared_ptr info_{new NiceMock()}; std::unique_ptr lb_; private: @@ -61,6 +62,12 @@ class LoadBalancerFuzzBase { // localities Key - index of host within full host list, value - locality level host at index is // in absl::node_hash_map locality_indexes_; + + static HostVector initializeHostsForUseInFuzzing(std::shared_ptr info); + + // Will statically initialize 60000 hosts in this vector. Will have to clear these static + // hosts flags at the end of each fuzz iteration + HostVector initialized_hosts_; }; } // namespace Upstream diff --git a/test/common/upstream/random_load_balancer_corpus/random_256_ports b/test/common/upstream/random_load_balancer_corpus/random_256_ports index 1924462a2ee7..e2792414cfe8 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_256_ports +++ b/test/common/upstream/random_load_balancer_corpus/random_256_ports @@ -6,7 +6,7 @@ actions { update_health_flags { host_priority: 0 num_healthy_hosts: 256 - random_bytestring: "\x01\x02\x03\x04\x45\x80" + random_bytestring: 1 } } actions { @@ -31,11 +31,11 @@ actions { } setup_priority_levels { num_hosts_in_priority_level: 256 - random_bytestring: "\x01\x02" + random_bytestring: 1 } setup_priority_levels { num_hosts_in_priority_level: 256 - random_bytestring: "\x01\x02" + random_bytestring: 1 } seed_for_prng: 4 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_NoHosts b/test/common/upstream/random_load_balancer_corpus/random_NoHosts index 551225e908e3..63b10ab1aa3d 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_NoHosts +++ b/test/common/upstream/random_load_balancer_corpus/random_NoHosts @@ -14,11 +14,17 @@ actions { } setup_priority_levels { num_hosts_in_priority_level: 0 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } setup_priority_levels { num_hosts_in_priority_level: 0 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } seed_for_prng: 2 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_Normal b/test/common/upstream/random_load_balancer_corpus/random_Normal index 61bb66f8638c..66bff38e17c6 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_Normal +++ b/test/common/upstream/random_load_balancer_corpus/random_Normal @@ -6,7 +6,10 @@ actions { update_health_flags { host_priority: 0 num_healthy_hosts: 2 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } } actions { @@ -31,11 +34,17 @@ actions { } setup_priority_levels { num_hosts_in_priority_level: 2 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } setup_priority_levels { num_hosts_in_priority_level: 0 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } seed_for_prng: 1 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_crash-55abbf82c64b5a62e299b93d7b254045471199c9 b/test/common/upstream/random_load_balancer_corpus/random_crash-55abbf82c64b5a62e299b93d7b254045471199c9 new file mode 100644 index 000000000000..ca01834b3116 --- /dev/null +++ b/test/common/upstream/random_load_balancer_corpus/random_crash-55abbf82c64b5a62e299b93d7b254045471199c9 @@ -0,0 +1,41 @@ +load_balancer_test_case { +common_lb_config { + +} +actions { + update_health_flags { + host_priority: 0 + num_healthy_hosts: 2 + random_bytestring: 1 + } +} +actions { + prefetch { + + } +} +actions { + prefetch { + + } +} +actions { + choose_host { + + } +} +actions { + choose_host { + + } +} +setup_priority_levels { + num_hosts_in_priority_level: 250 + random_bytestring: 1 +} +setup_priority_levels { + num_hosts_in_priority_level: 250 + random_bytestring: 1 +} +seed_for_prng: 4 +} diff --git a/test/common/upstream/random_load_balancer_corpus/random_crash-5a64c214e5c2038299c8f2e85f143f1163077c1b b/test/common/upstream/random_load_balancer_corpus/random_crash-5a64c214e5c2038299c8f2e85f143f1163077c1b new file mode 100644 index 000000000000..cb19f383c387 --- /dev/null +++ b/test/common/upstream/random_load_balancer_corpus/random_crash-5a64c214e5c2038299c8f2e85f143f1163077c1b @@ -0,0 +1,25 @@ +load_balancer_test_case { + common_lb_config { + healthy_panic_threshold { + value: 4.88907830238399e-311 + } + consistent_hashing_lb_config { + use_hostname_for_hashing: true + hash_balance_factor { + value: 1024 + } + } + } + actions { + update_health_flags { + host_priority: 270582939648 + num_degraded_hosts: 4194304 + random_bytestring: 1 + } + } + setup_priority_levels { + num_hosts_in_priority_level: 1024 + random_bytestring: 1 + } + seed_for_prng: 62208 +} diff --git a/test/common/upstream/random_load_balancer_corpus/random_crash-ba5efdfd9c412a8507087120783fe6529b1ac0cb b/test/common/upstream/random_load_balancer_corpus/random_crash-ba5efdfd9c412a8507087120783fe6529b1ac0cb index 602a393132bf..65c8062d59c0 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_crash-ba5efdfd9c412a8507087120783fe6529b1ac0cb +++ b/test/common/upstream/random_load_balancer_corpus/random_crash-ba5efdfd9c412a8507087120783fe6529b1ac0cb @@ -28,11 +28,11 @@ actions { } setup_priority_levels { num_hosts_in_priority_level: 2 - random_bytestring: "\x01\x02" + random_bytestring: 1 } setup_priority_levels { num_hosts_in_priority_level: 9007199259945536 - random_bytestring: "\x01\x02" + random_bytestring: 1 } seed_for_prng: 6 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_largest-port-value b/test/common/upstream/random_load_balancer_corpus/random_largest-port-value index 2f95ce787ef1..a89ecba1b8de 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_largest-port-value +++ b/test/common/upstream/random_load_balancer_corpus/random_largest-port-value @@ -24,11 +24,11 @@ actions { } nsetup_priority_levels { num_hosts_in_priority_level: 65455 - random_bytestring: "\x01\x02" + random_bytestring: 1 } setup_priority_levels { num_hosts_in_priority_level: 65455 - random_bytestring: "\x01\x02" + random_bytestring: 1 } seed_for_prng: 5 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_many_choose_hosts b/test/common/upstream/random_load_balancer_corpus/random_many_choose_hosts index b263d07ec40e..53a69c852f58 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_many_choose_hosts +++ b/test/common/upstream/random_load_balancer_corpus/random_many_choose_hosts @@ -51,11 +51,15 @@ actions { } setup_priority_levels { num_hosts_in_priority_level: 2 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 } setup_priority_levels { num_hosts_in_priority_level: 0 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } seed_for_prng: 1 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_max_ports b/test/common/upstream/random_load_balancer_corpus/random_max_ports index 4a7406d8b765..73e56d8d7637 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_max_ports +++ b/test/common/upstream/random_load_balancer_corpus/random_max_ports @@ -6,7 +6,10 @@ actions { update_health_flags { host_priority: 0 num_healthy_hosts: 2 - random_bytestring: "\x01\x02\x03\x04" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 3 + random_bytestring: 4 } } actions { @@ -31,11 +34,17 @@ actions { } setup_priority_levels { num_hosts_in_priority_level: 32726 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } setup_priority_levels { num_hosts_in_priority_level: 32726 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } seed_for_prng: 88 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_overflowing_ports b/test/common/upstream/random_load_balancer_corpus/random_overflowing_ports index 4598c29dbe10..4ec8cd27d2ac 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_overflowing_ports +++ b/test/common/upstream/random_load_balancer_corpus/random_overflowing_ports @@ -6,7 +6,10 @@ actions { update_health_flags { host_priority: 0 num_healthy_hosts: 2 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } } actions { @@ -31,11 +34,17 @@ actions { } setup_priority_levels { num_hosts_in_priority_level: 60000 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } setup_priority_levels { num_hosts_in_priority_level: 60000 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } seed_for_prng: 4 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_slow-unit-eed4596101efb3e737f736c8d5bcd4f0815a8728 b/test/common/upstream/random_load_balancer_corpus/random_slow-unit-eed4596101efb3e737f736c8d5bcd4f0815a8728 index 7bebf1a2cf96..b8f81a4d451c 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_slow-unit-eed4596101efb3e737f736c8d5bcd4f0815a8728 +++ b/test/common/upstream/random_load_balancer_corpus/random_slow-unit-eed4596101efb3e737f736c8d5bcd4f0815a8728 @@ -7,7 +7,10 @@ load_balancer_test_case { actions { update_health_flags { num_healthy_hosts: 2 - random_bytestring: "\001\002\003\004" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 3 + random_bytestring: 4 } } actions { @@ -25,16 +28,25 @@ load_balancer_test_case { actions { update_health_flags { num_healthy_hosts: 2 - random_bytestring: "\001\002\003\004" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 3 + random_bytestring: 4 } } setup_priority_levels { num_hosts_in_priority_level: 536903638 - random_bytestring: "\001\002" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 3 + random_bytestring: 4 } setup_priority_levels { num_hosts_in_priority_level: 32726 - random_bytestring: "\001\002" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 3 + random_bytestring: 4 } seed_for_prng: 88 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_slow-unit-test b/test/common/upstream/random_load_balancer_corpus/random_slow-unit-test index e1f2fcfdd303..1abe2824c65b 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_slow-unit-test +++ b/test/common/upstream/random_load_balancer_corpus/random_slow-unit-test @@ -7,7 +7,10 @@ load_balancer_test_case { actions { update_health_flags { num_healthy_hosts: 2 - random_bytestring: "\001\002\003\004" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } } actions { @@ -25,7 +28,10 @@ load_balancer_test_case { actions { update_health_flags { num_healthy_hosts: 2 - random_bytestring: "\001\002\003\004" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } } setup_priority_levels { @@ -33,14 +39,20 @@ load_balancer_test_case { num_hosts_locality_one: 50 num_hosts_locality_two: 50 num_hosts_locality_three: 50 - random_bytestring: "\001\002" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } setup_priority_levels { num_hosts_in_priority_level: 500 num_hosts_locality_one: 50 num_hosts_locality_two: 50 num_hosts_locality_three: 50 - random_bytestring: "\001\002" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } seed_for_prng: 88 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_test_something b/test/common/upstream/random_load_balancer_corpus/random_test_something index 172e2cb1c051..7025e0fed767 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_test_something +++ b/test/common/upstream/random_load_balancer_corpus/random_test_something @@ -6,7 +6,10 @@ actions { update_health_flags { host_priority: 0 num_healthy_hosts: 2 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } } actions { @@ -31,11 +34,17 @@ actions { } setup_priority_levels { num_hosts_in_priority_level: 250 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } setup_priority_levels { num_hosts_in_priority_level: 250 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } seed_for_prng: 4 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_timeout-6b0d6b83136a4cf0b9ccd468f11207a792859d43 b/test/common/upstream/random_load_balancer_corpus/random_timeout-6b0d6b83136a4cf0b9ccd468f11207a792859d43 index 96f3d0efd72b..75df003cee29 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_timeout-6b0d6b83136a4cf0b9ccd468f11207a792859d43 +++ b/test/common/upstream/random_load_balancer_corpus/random_timeout-6b0d6b83136a4cf0b9ccd468f11207a792859d43 @@ -20,12 +20,18 @@ load_balancer_test_case { actions { update_health_flags { num_excluded_hosts: 268435456 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } } setup_priority_levels { num_hosts_in_priority_level: 13534154135 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } seed_for_prng: 32 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_timeout-9144cfbb40b5101ecc28b205b10e6c36a72aae83 b/test/common/upstream/random_load_balancer_corpus/random_timeout-9144cfbb40b5101ecc28b205b10e6c36a72aae83 index 2fca35ed475d..79d6fde49fde 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_timeout-9144cfbb40b5101ecc28b205b10e6c36a72aae83 +++ b/test/common/upstream/random_load_balancer_corpus/random_timeout-9144cfbb40b5101ecc28b205b10e6c36a72aae83 @@ -14,12 +14,18 @@ load_balancer_test_case { update_health_flags { host_priority: 270582939648 num_degraded_hosts: 4194304 - random_bytestring: "\x01\x02\x03\x04" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } } setup_priority_levels { num_hosts_in_priority_level: 1024 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } seed_for_prng: 62208 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_with-locality b/test/common/upstream/random_load_balancer_corpus/random_with-locality index 3f2f1281845f..15adcc4de667 100644 --- a/test/common/upstream/random_load_balancer_corpus/random_with-locality +++ b/test/common/upstream/random_load_balancer_corpus/random_with-locality @@ -8,7 +8,10 @@ actions { num_healthy_hosts: 2 num_degraded_hosts: 3 num_excluded_hosts: 4 - random_bytestring: "\x01\x02\x03\x04\x05\x06" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } } actions { @@ -36,14 +39,20 @@ setup_priority_levels { num_hosts_locality_a: 3 num_hosts_locality_b: 4 num_hosts_locality_c: 5 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } setup_priority_levels { num_hosts_in_priority_level: 20 num_hosts_locality_a: 3 num_hosts_locality_b: 4 num_hosts_locality_c: 5 - random_bytestring: "\x01\x02" + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 } seed_for_prng: 1 } diff --git a/test/common/upstream/random_load_balancer_corpus/random_with-locality-high-number-of-hosts b/test/common/upstream/random_load_balancer_corpus/random_with-locality-high-number-of-hosts new file mode 100644 index 000000000000..2a96d688b5d6 --- /dev/null +++ b/test/common/upstream/random_load_balancer_corpus/random_with-locality-high-number-of-hosts @@ -0,0 +1,58 @@ +load_balancer_test_case { +common_lb_config { + +} +actions { + update_health_flags { + host_priority: 0 + num_healthy_hosts: 2 + num_degraded_hosts: 3 + num_excluded_hosts: 4 + random_bytestring: 100000 + random_bytestring: 1000000 + random_bytestring: 1500000 + random_bytestring: 2000000 + } +} +actions { + prefetch { + + } +} +actions { + prefetch { + + } +} +actions { + choose_host { + + } +} +actions { + choose_host { + + } +} +setup_priority_levels { + num_hosts_in_priority_level: 3000 + num_hosts_locality_a: 1000 + num_hosts_locality_b: 500 + num_hosts_locality_c: 1500 + random_bytestring: 100000 + random_bytestring: 1000000 + random_bytestring: 1500000 + random_bytestring: 2000000 +} +setup_priority_levels { + num_hosts_in_priority_level: 3000 + num_hosts_locality_a: 300 + num_hosts_locality_b: 1200 + num_hosts_locality_c: 1500 + random_bytestring: 100000 + random_bytestring: 1000000 + random_bytestring: 1500000 + random_bytestring: 2000000 +} +seed_for_prng: 1 +} diff --git a/test/common/upstream/random_load_balancer_corpus/random_with_locality-50000-hosts b/test/common/upstream/random_load_balancer_corpus/random_with_locality-50000-hosts new file mode 100644 index 000000000000..7a1784375405 --- /dev/null +++ b/test/common/upstream/random_load_balancer_corpus/random_with_locality-50000-hosts @@ -0,0 +1,58 @@ +load_balancer_test_case { +common_lb_config { + +} +actions { + update_health_flags { + host_priority: 0 + num_healthy_hosts: 2 + num_degraded_hosts: 3 + num_excluded_hosts: 4 + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 + } +} +actions { + prefetch { + + } +} +actions { + prefetch { + + } +} +actions { + choose_host { + + } +} +actions { + choose_host { + + } +} +setup_priority_levels { + num_hosts_in_priority_level: 50000 + num_hosts_locality_a: 1000 + num_hosts_locality_b: 500 + num_hosts_locality_c: 1500 + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 +} +setup_priority_levels { + num_hosts_in_priority_level: 50000 + num_hosts_locality_a: 300 + num_hosts_locality_b: 1200 + num_hosts_locality_c: 1500 + random_bytestring: 1 + random_bytestring: 2 + random_bytestring: 1 + random_bytestring: 2 +} +seed_for_prng: 1 +} diff --git a/test/fuzz/random.h b/test/fuzz/random.h index 01b8f5e83df5..d6528964f858 100644 --- a/test/fuzz/random.h +++ b/test/fuzz/random.h @@ -33,9 +33,10 @@ class PsuedoRandomGenerator64 : public RandomGenerator { } // namespace Random namespace Fuzz { + class ProperSubsetSelector { public: - ProperSubsetSelector(const std::string& random_bytestring) + ProperSubsetSelector(const std::vector& random_bytestring) : random_bytestring_(random_bytestring) {} /** @@ -50,16 +51,16 @@ class ProperSubsetSelector { * of elements, the function would return something such as {{5, 3}}. */ - std::vector> + std::vector> constructSubsets(const std::vector& number_of_elements_in_each_subset, uint32_t number_of_elements) { num_elements_left_ = number_of_elements; - std::vector index_vector; + std::vector index_vector; index_vector.reserve(number_of_elements); for (uint32_t i = 0; i < number_of_elements; i++) { index_vector.push_back(i); } - std::vector> subsets; + std::vector> subsets; subsets.reserve(number_of_elements_in_each_subset.size()); for (uint32_t i : number_of_elements_in_each_subset) { subsets.push_back(constructSubset(i, index_vector)); @@ -69,16 +70,15 @@ class ProperSubsetSelector { private: // Builds a single subset by pulling indexes off index_vector_ - std::vector constructSubset(uint32_t number_of_elements_in_subset, - std::vector& index_vector) { - std::vector subset; + std::vector constructSubset(uint32_t number_of_elements_in_subset, + std::vector& index_vector) { + std::vector subset; for (uint32_t i = 0; i < number_of_elements_in_subset && !(num_elements_left_ == 0); i++) { - // Index of bytestring will wrap around if it "overflows" past the random bytestring's length. - uint64_t index_of_index_vector = - random_bytestring_[index_of_random_bytestring_ % random_bytestring_.length()] % + uint32_t index_of_index_vector = + random_bytestring_.at(index_of_random_bytestring_ % random_bytestring_.size()) % num_elements_left_; - const uint64_t index = index_vector.at(index_of_index_vector); + uint32_t index = index_vector.at(index_of_index_vector); subset.push_back(index); // Move the index chosen to the end of the vector - will not be chosen again std::swap(index_vector[index_of_index_vector], index_vector[num_elements_left_ - 1]); @@ -92,7 +92,7 @@ class ProperSubsetSelector { // This bytestring will be iterated through representing randomness in order to choose // subsets - const std::string random_bytestring_; + std::vector random_bytestring_; uint32_t index_of_random_bytestring_ = 0; // Used to make subset construction linear time complexity with std::swap - chosen indexes will be diff --git a/test/fuzz/random_test.cc b/test/fuzz/random_test.cc index 9e7fd1012260..4613583d468d 100644 --- a/test/fuzz/random_test.cc +++ b/test/fuzz/random_test.cc @@ -9,16 +9,18 @@ namespace Envoy { namespace Fuzz { // Test the subset selection - since selection is based on a passed in random bytestring you can -// work the algorithm yourself Pass in 5 elements, expect first subset to be element 2 and element -// 4, second subset to be elements 1, 2, 3 -TEST(BasicSubsetSelection, RandomTest) { - // \x01 chooses the first element, which gets swapped with last element, 0x3 chooses the third - // index, which gets swapped with last element etc. - std::string random_bytestring = "\x01\x03\x09\x04\x33"; +// work the algorithm yourself. This test also tests if the subset selection handles 32 bits. +TEST(BasicSubsetSelection, ValidateScaleTest) { + std::vector random_bytestring; + random_bytestring.push_back(1000000); + random_bytestring.push_back(1500000); + random_bytestring.push_back(2000000); + random_bytestring.push_back(2500000); ProperSubsetSelector subset_selector(random_bytestring); - const std::vector> subsets = subset_selector.constructSubsets({2, 3}, 5); - const std::vector expected_subset_one = {1, 3}; - const std::vector expected_subset_two = {0, 2, 4}; + const std::vector> subsets = + subset_selector.constructSubsets({2, 2}, 10000000); + const std::vector expected_subset_one = {1000000, 1500000}; + const std::vector expected_subset_two = {2000000, 2500000}; EXPECT_THAT(subsets[0], ContainerEq(expected_subset_one)); EXPECT_THAT(subsets[1], ContainerEq(expected_subset_two)); } From 0441602b659ce9f672a2e21de8d53c63e98dc3e2 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Fri, 30 Oct 2020 10:46:37 -0400 Subject: [PATCH 6/9] test: adding a test for CONNECT to an IP address (#13818) Signed-off-by: Alyssa Wilk --- .../tcp_tunneling_integration_test.cc | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/integration/tcp_tunneling_integration_test.cc b/test/integration/tcp_tunneling_integration_test.cc index 00efaba709a6..efc7222c90cb 100644 --- a/test/integration/tcp_tunneling_integration_test.cc +++ b/test/integration/tcp_tunneling_integration_test.cc @@ -272,6 +272,41 @@ TEST_P(ProxyingConnectIntegrationTest, ProxyConnect) { cleanupUpstreamAndDownstream(); } +TEST_P(ProxyingConnectIntegrationTest, ProxyConnectWithIP) { + initialize(); + + // Send request headers. + codec_client_ = makeHttpConnection(lookupPort("http")); + connect_headers_.setHost("1.2.3.4:80"); + auto encoder_decoder = codec_client_->startRequest(connect_headers_); + request_encoder_ = &encoder_decoder.first; + response_ = std::move(encoder_decoder.second); + + // Wait for them to arrive upstream. + AssertionResult result = + fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_); + RELEASE_ASSERT(result, result.message()); + result = fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_); + RELEASE_ASSERT(result, result.message()); + ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); + EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Method)[0]->value(), "CONNECT"); + if (upstreamProtocol() == FakeHttpConnection::Type::HTTP1) { + EXPECT_TRUE(upstream_request_->headers().get(Http::Headers::get().Protocol).empty()); + } else { + EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Protocol)[0]->value(), + "bytestream"); + } + + // Send response headers + upstream_request_->encodeHeaders(default_response_headers_, false); + + // Wait for them to arrive downstream. + response_->waitForHeaders(); + EXPECT_EQ("200", response_->headers().getStatusValue()); + + cleanupUpstreamAndDownstream(); +} + INSTANTIATE_TEST_SUITE_P(IpVersions, ConnectTerminationIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); From 7ab7de1853e436a282c6a261b4616d48e96cb1e8 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Fri, 30 Oct 2020 08:31:49 -0700 Subject: [PATCH 7/9] wasm: update V8 to v8.7.220.10. (#13568) Signed-off-by: Piotr Sikora --- bazel/external/wee8.BUILD | 11 +++++++---- bazel/external/wee8.patch | 19 ++++++++++++++++--- bazel/repository_locations.bzl | 6 +++--- test/tools/wee8_compile/wee8_compile.cc | 12 ++++++++---- tools/code_format/check_format.py | 2 +- 5 files changed, 35 insertions(+), 15 deletions(-) diff --git a/bazel/external/wee8.BUILD b/bazel/external/wee8.BUILD index 3a62ecd9ebf4..ce16c32af799 100644 --- a/bazel/external/wee8.BUILD +++ b/bazel/external/wee8.BUILD @@ -9,15 +9,18 @@ cc_library( srcs = [ "libwee8.a", ], - hdrs = [ - "wee8/include/v8-version.h", - "wee8/third_party/wasm-api/wasm.hh", - ], + hdrs = + glob([ + "wee8/include/**/*.h", + "wee8/src/**/*.h", + "wee8/third_party/wasm-api/wasm.hh", + ]), copts = [ "-Wno-range-loop-analysis", ], defines = ["ENVOY_WASM_V8"], includes = [ + "wee8", "wee8/include", "wee8/third_party", ], diff --git a/bazel/external/wee8.patch b/bazel/external/wee8.patch index cce3eecde614..50255793070e 100644 --- a/bazel/external/wee8.patch +++ b/bazel/external/wee8.patch @@ -1,9 +1,10 @@ # 1. Fix linking with unbundled toolchain on macOS. # 2. Increase VSZ limit to 4TiB (allows us to start up to 409 VMs). # 3. Fix MSAN linking. +# 4. Fix Wasm module deserialization (http://crbug.com/v8/11024). --- wee8/build/toolchain/gcc_toolchain.gni +++ wee8/build/toolchain/gcc_toolchain.gni -@@ -329,6 +329,8 @@ template("gcc_toolchain") { +@@ -348,6 +348,8 @@ template("gcc_toolchain") { # AIX does not support either -D (deterministic output) or response # files. command = "$ar -X64 {{arflags}} -r -c -s {{output}} {{inputs}}" @@ -12,7 +13,7 @@ } else { rspfile = "{{output}}.rsp" rspfile_content = "{{inputs}}" -@@ -507,7 +509,7 @@ template("gcc_toolchain") { +@@ -543,7 +545,7 @@ template("gcc_toolchain") { start_group_flag = "" end_group_flag = "" @@ -51,5 +52,17 @@ - is_msan && (msan_track_origins == 0 || msan_track_origins == 2) +prebuilt_instrumented_libraries_available = false - if (use_libfuzzer && is_linux) { + if (use_libfuzzer && (is_linux || is_chromeos)) { if (is_asan) { +--- wee8/src/wasm/module-compiler.cc ++++ wee8/src/wasm/module-compiler.cc +@@ -2901,6 +2901,9 @@ void CompilationStateImpl::InitializeCompilationProgressAfterDeserialization() { + RequiredBaselineTierField::encode(ExecutionTier::kTurbofan) | + RequiredTopTierField::encode(ExecutionTier::kTurbofan) | + ReachedTierField::encode(ExecutionTier::kTurbofan); ++ finished_events_.Add(CompilationEvent::kFinishedExportWrappers); ++ finished_events_.Add(CompilationEvent::kFinishedBaselineCompilation); ++ finished_events_.Add(CompilationEvent::kFinishedTopTierCompilation); + compilation_progress_.assign(module->num_declared_functions, + kProgressAfterDeserialization); + } diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index c9a3dbd343c9..974c9deea62b 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -651,10 +651,10 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "V8", project_desc = "Google’s open source high-performance JavaScript and WebAssembly engine, written in C++", project_url = "https://v8.dev", - version = "8.5.210.20", + version = "8.7.220.10", # This archive was created using https://storage.googleapis.com/envoyproxy-wee8/wee8-archive.sh # and contains complete checkout of V8 with all dependencies necessary to build wee8. - sha256 = "ef404643d7da6854b76b9fb9950a79a1acbd037b7a26f02c585ac379b0f7dee1", + sha256 = "f22734640e0515bc34d1ca3772513aef24374fafa44d0489d3a9a57cadec69fb", urls = ["https://storage.googleapis.com/envoyproxy-wee8/wee8-{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = [ @@ -664,7 +664,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", ], - release_date = "2020-08-17", + release_date = "2020-10-27", cpe = "cpe:2.3:a:google:v8:*", ), com_googlesource_quiche = dict( diff --git a/test/tools/wee8_compile/wee8_compile.cc b/test/tools/wee8_compile/wee8_compile.cc index a1b2906ab281..fe257a95e1a6 100644 --- a/test/tools/wee8_compile/wee8_compile.cc +++ b/test/tools/wee8_compile/wee8_compile.cc @@ -1,12 +1,13 @@ // NOLINT(namespace-envoy) -#include - +#include #include #include #include +#include #include +#include "src/wasm/c-api.h" #include "v8-version.h" #include "wasm-api/wasm.hh" @@ -166,8 +167,11 @@ wasm::vec serializeWasmModule(const char* path, const wasm::vec& return wasm::vec::invalid(); } - // TODO(PiotrSikora): figure out how to hook the completion callback. - sleep(3); + wasm::StoreImpl* store_impl = reinterpret_cast(store.get()); + auto isolate = store_impl->isolate(); + while (isolate->HasPendingBackgroundTasks()) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } return module->serialize(); } diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index bb25f20cba44..ebe10e9ba133 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -58,7 +58,7 @@ "./test/test_common/simulated_time_system.cc", "./test/test_common/simulated_time_system.h", "./test/test_common/test_time.cc", "./test/test_common/test_time.h", "./test/test_common/utility.cc", "./test/test_common/utility.h", - "./test/integration/integration.h") + "./test/integration/integration.h", "./test/tools/wee8_compile/wee8_compile.cc") # Tests in these paths may make use of the Registry::RegisterFactory constructor or the # REGISTER_FACTORY macro. Other locations should use the InjectFactory helper class to From c36abec5e6e88cd7e2aadcdc89cafc52d8bb8cb2 Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 30 Oct 2020 15:39:53 +0000 Subject: [PATCH 8/9] docs: Further updates to quick-start (#13793) Signed-off-by: Ryan Northey --- docs/BUILD | 1 + docs/redirects.txt | 2 + docs/root/start/quick-start.rst | 240 ------------------ .../_include/envoy-demo.yaml | 0 .../_include/envoy-dynamic-cds-demo.yaml | 20 ++ .../envoy-dynamic-control-plane-demo.yaml | 40 +++ .../envoy-dynamic-filesystem-demo.yaml | 16 ++ .../_include/envoy-dynamic-lds-demo.yaml | 27 ++ .../configuration-dynamic-control-plane.rst | 97 +++++++ .../configuration-dynamic-filesystem.rst | 115 +++++++++ .../quick-start/configuration-static.rst | 80 ++++++ docs/root/start/quick-start/index.rst | 16 ++ docs/root/start/quick-start/next-steps.rst | 23 ++ docs/root/start/quick-start/run-envoy.rst | 160 ++++++++++++ docs/root/start/start.rst | 2 +- examples/dynamic-config-cp/envoy.yaml | 9 - 16 files changed, 598 insertions(+), 250 deletions(-) delete mode 100644 docs/root/start/quick-start.rst rename docs/root/start/{ => quick-start}/_include/envoy-demo.yaml (100%) create mode 100644 docs/root/start/quick-start/_include/envoy-dynamic-cds-demo.yaml create mode 100644 docs/root/start/quick-start/_include/envoy-dynamic-control-plane-demo.yaml create mode 100644 docs/root/start/quick-start/_include/envoy-dynamic-filesystem-demo.yaml create mode 100644 docs/root/start/quick-start/_include/envoy-dynamic-lds-demo.yaml create mode 100644 docs/root/start/quick-start/configuration-dynamic-control-plane.rst create mode 100644 docs/root/start/quick-start/configuration-dynamic-filesystem.rst create mode 100644 docs/root/start/quick-start/configuration-static.rst create mode 100644 docs/root/start/quick-start/index.rst create mode 100644 docs/root/start/quick-start/next-steps.rst create mode 100644 docs/root/start/quick-start/run-envoy.rst diff --git a/docs/BUILD b/docs/BUILD index aad5c89f0b65..65323e28015b 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -17,6 +17,7 @@ filegroup( "root/**/*.pb", ], exclude = [ + "root/**/envoy-dynamic*.yaml", # TODO(phlax/windows-dev): figure out how to get this working on windows # "Error: unable to read file: /etc/ssl/certs/ca-certificates.crt" "root/configuration/http/http_filters/_include/dns-cache-circuit-breaker.yaml", diff --git a/docs/redirects.txt b/docs/redirects.txt index c4d2cb39faac..e646e88d507a 100644 --- a/docs/redirects.txt +++ b/docs/redirects.txt @@ -17,3 +17,5 @@ start/install/tools/tools.rst operations/tools/tools.rst start/install/tools/route_table_check_tool.rst operations/tools/route_table_check_tool.rst start/install/tools/schema_validator_check_tool.rst operations/tools/schema_validator_check_tool.rst start/install/tools/config_load_check_tool.rst operations/tools/config_load_check_tool.rst + +start/quick-start.rst start/quick-start/index.rst diff --git a/docs/root/start/quick-start.rst b/docs/root/start/quick-start.rst deleted file mode 100644 index 319832c71e91..000000000000 --- a/docs/root/start/quick-start.rst +++ /dev/null @@ -1,240 +0,0 @@ -.. _start_quick_start: - - -Quick start -=========== - -The following instructions walk through starting Envoy as a system daemon or using -the Envoy Docker image. - -.. _start_quick_start_version: - -Check your Envoy version ------------------------- - -Once you have :ref:`installed Envoy `, you can check the version information as follows: - -.. tabs:: - - .. tab:: System - - .. code-block:: console - - $ envoy --version - - .. tab:: Docker - - .. substitution-code-block:: console - - $ docker run --rm envoyproxy/|envoy_docker_image| --version - -.. _start_quick_start_help: - -View the Envoy command line options ------------------------------------ - -You can view the Envoy :ref:`command line options ` with the ``--help`` -flag: - -.. tabs:: - - .. tab:: System - - .. code-block:: console - - $ envoy --help - - .. tab:: Docker - - .. substitution-code-block:: console - - $ docker run --rm envoyproxy/|envoy_docker_image| --help - -.. _start_quick_start_config: - -Run Envoy with the demo configuration -------------------------------------- - -The ``-c`` or ``--config-path`` flag tells Envoy the path to its initial configuration. - -.. tabs:: - - .. tab:: System - - To start Envoy as a system daemon :download:`download the demo configuration <_include/envoy-demo.yaml>`, and start - as follows: - - .. code-block:: console - - $ envoy -c envoy-demo.yaml - - .. tab:: Docker - - You can start the Envoy Docker image without specifying a configuration file, and - it will use the demo config by default. - - .. substitution-code-block:: console - - $ docker run --rm -d -p 9901:9901 -p 10000:10000 envoyproxy/|envoy_docker_image| - - To specify a custom configuration you can mount the config into the container, and specify the path with ``-c``. - - Assuming you have a custom configuration in the current directory named ``envoy-custom.yaml``: - - .. substitution-code-block:: console - - $ docker run --rm -d -v $(pwd)/envoy-custom.yaml:/envoy-custom.yaml -p 9901:9901 -p 10000:10000 envoyproxy/|envoy_docker_image| -c /envoy-custom.yaml - -Check Envoy is proxying on http://localhost:10000 - -.. code-block:: console - - $ curl -v localhost:10000 - -The Envoy admin endpoint should also be available at http://localhost:9901 - -.. code-block:: console - - $ curl -v localhost:9901 - -.. _start_quick_start_override: - -Override the default configuration by merging a config file ------------------------------------------------------------ - -You can provide a configuration override file using ``--config-yaml`` which will merge with the main -configuration. - -Save the following snippet to ``envoy-override.yaml``: - -.. code-block:: yaml - - listeners: - - name: listener_0 - address: - socket_address: - port_value: 20000 - -Next, start the Envoy server using the override configuration. - -.. tabs:: - - .. tab:: System - - .. code-block:: console - - $ envoy -c envoy-demo.yaml --config-yaml envoy-override.yaml - - .. tab:: Docker - - .. substitution-code-block:: console - - $ docker run --rm -d -v $(pwd)/envoy-override.yaml:/envoy-override.yaml -p 20000:20000 envoyproxy/|envoy_docker_image| --config-yaml /envoy-override.yaml - -Envoy should now be proxying on http://localhost:20000 - -.. code-block:: console - - $ curl -v localhost:20000 - -The Envoy admin endpoint should also be available at http://localhost:9901 - -.. code-block:: console - - $ curl -v localhost:9901 - -.. _start_quick_start_static: - -Static configuration --------------------- - -To start Envoy with static configuration, you will need to specify :ref:`listeners ` -and :ref:`clusters ` as -:ref:`static_resources `. - -You can also add an :ref:`admin ` section if you wish to monitor Envoy -or retrieve stats. - -The following sections walk through the static configuration provided in the -:download:`demo configuration file <_include/envoy-demo.yaml>` used as the default in the Envoy Docker container. - -.. _start_quick_start_static_static_resources: - -Static configuration: ``static_resources`` -****************************************** - -The :ref:`static_resources ` contain -everything that is configured statically when Envoy starts, as opposed to dynamically at runtime. - -.. literalinclude:: _include/envoy-demo.yaml - :language: yaml - :linenos: - :lines: 1-3 - :emphasize-lines: 1 - -.. _start_quick_start_static_listeners: - -Static configuration: ``listeners`` -*********************************** - -The example configures a :ref:`listener ` -on port ``10000``. - -All paths are matched and routed to the ``service_envoyproxy_io`` -:ref:`cluster `. - -.. literalinclude:: _include/envoy-demo.yaml - :language: yaml - :linenos: - :lines: 1-29 - :emphasize-lines: 3-27 - -.. _start_quick_start_static_clusters: - -Static configuration: ``clusters`` -********************************** - -The ``service_envoyproxy_io`` :ref:`cluster ` -proxies over ``TLS`` to https://www.envoyproxy.io. - -.. literalinclude:: _include/envoy-demo.yaml - :language: yaml - :lineno-start: 27 - :lines: 27-50 - :emphasize-lines: 3-22 - -.. _start_quick_start_static_admin: - -Static configuration: ``admin`` -******************************* - -The :ref:`admin message ` is required to enable and configure -the administration server. - -The ``address`` key specifies the listening :ref:`address ` -which in the demo configuration is ``0.0.0.0:9901``. - -.. literalinclude:: _include/envoy-demo.yaml - :language: yaml - :lineno-start: 48 - :lines: 48-55 - :emphasize-lines: 3-8 - -.. warning:: - - You may wish to restrict the network address the admin server listens to in your own deployment. - -.. _start_quick_start_dynamic: - -Dynamic configuration ---------------------- - -See the :ref:`configuration overview ` for further information on configuring Envoy with static and dynamic configuration. - -Next steps ----------- - -- Learn more about :ref:`using the Envoy Docker image ` -- Try out demo configurations in the :ref:`sandboxes ` -- Check out the :ref:`configuration generator ` and other - :ref:`Envoy tools ` diff --git a/docs/root/start/_include/envoy-demo.yaml b/docs/root/start/quick-start/_include/envoy-demo.yaml similarity index 100% rename from docs/root/start/_include/envoy-demo.yaml rename to docs/root/start/quick-start/_include/envoy-demo.yaml diff --git a/docs/root/start/quick-start/_include/envoy-dynamic-cds-demo.yaml b/docs/root/start/quick-start/_include/envoy-dynamic-cds-demo.yaml new file mode 100644 index 000000000000..9a4d656eeb83 --- /dev/null +++ b/docs/root/start/quick-start/_include/envoy-dynamic-cds-demo.yaml @@ -0,0 +1,20 @@ +resources: +- "@type": type.googleapis.com/envoy.config.cluster.v3.Cluster + name: example_proxy_cluster + connect_timeout: 1s + type: strict_dns + http2_protocol_options: {} + load_assignment: + cluster_name: example_proxy_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: www.envoyproxy.io + port_value: 443 + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + sni: www.envoyproxy.io diff --git a/docs/root/start/quick-start/_include/envoy-dynamic-control-plane-demo.yaml b/docs/root/start/quick-start/_include/envoy-dynamic-control-plane-demo.yaml new file mode 100644 index 000000000000..e1963a104ff4 --- /dev/null +++ b/docs/root/start/quick-start/_include/envoy-dynamic-control-plane-demo.yaml @@ -0,0 +1,40 @@ +node: + cluster: test-cluster + id: test-id + +dynamic_resources: + ads_config: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + cds_config: + resource_api_version: V3 + ads: {} + lds_config: + resource_api_version: V3 + ads: {} + +static_resources: + clusters: + - connect_timeout: 1s + type: strict_dns + http2_protocol_options: {} + name: xds_cluster + load_assignment: + cluster_name: xds_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: my-control-plane + port_value: 18000 + +admin: + access_log_path: /dev/null + address: + socket_address: + address: 0.0.0.0 + port_value: 19000 diff --git a/docs/root/start/quick-start/_include/envoy-dynamic-filesystem-demo.yaml b/docs/root/start/quick-start/_include/envoy-dynamic-filesystem-demo.yaml new file mode 100644 index 000000000000..728ceeddc8e5 --- /dev/null +++ b/docs/root/start/quick-start/_include/envoy-dynamic-filesystem-demo.yaml @@ -0,0 +1,16 @@ +node: + cluster: test-cluster + id: test-id + +dynamic_resources: + cds_config: + path: /var/lib/envoy/cds.yaml + lds_config: + path: /var/lib/envoy/lds.yaml + +admin: + access_log_path: "/dev/null" + address: + socket_address: + address: 0.0.0.0 + port_value: 19000 diff --git a/docs/root/start/quick-start/_include/envoy-dynamic-lds-demo.yaml b/docs/root/start/quick-start/_include/envoy-dynamic-lds-demo.yaml new file mode 100644 index 000000000000..1d07e44770b2 --- /dev/null +++ b/docs/root/start/quick-start/_include/envoy-dynamic-lds-demo.yaml @@ -0,0 +1,27 @@ +resources: +- "@type": type.googleapis.com/envoy.config.listener.v3.Listener + name: listener_0 + address: + socket_address: + address: 0.0.0.0 + port_value: 10000 + filter_chains: + - filters: + name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + http_filters: + - name: envoy.router + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + host_rewrite_literal: www.envoyproxy.io + cluster: example_proxy_cluster diff --git a/docs/root/start/quick-start/configuration-dynamic-control-plane.rst b/docs/root/start/quick-start/configuration-dynamic-control-plane.rst new file mode 100644 index 000000000000..3ceb430abe36 --- /dev/null +++ b/docs/root/start/quick-start/configuration-dynamic-control-plane.rst @@ -0,0 +1,97 @@ +.. _start_quick_start_dynamic_control_plane: + +Configuration: Dynamic from control plane +========================================= + +These instructions are slightly more complex as you must also set up a control plane to provide Envoy with its +configuration. + +There are a number of control planes compatible with Envoy's API such as `Gloo `_ +or `Istio `_. + +You may also wish to explore implementing your own control plane, in which case the +`Go Control Plane `_ provides a reference implementation +that is a good place to start. + +At a minimum, you will need to start Envoy configured with the following sections: + +- :ref:`node ` to uniquely identify the proxy node. +- :ref:`dynamic_resources ` to tell Envoy which configurations should be updated dynamically +- :ref:`static_resources ` to specify where Envoy should retrieve its configuration from. + +You can also add an :ref:`admin ` section if you wish to monitor Envoy or +retrieve stats or configuration information. + +The following sections walk through the dynamic configuration provided in the +:download:`demo dynamic control plane configuration file <_include/envoy-dynamic-control-plane-demo.yaml>`. + +.. _start_quick_start_dynamic_node: + +``node`` +-------- + +The :ref:`node ` should specify ``cluster`` and ``id``. + +.. literalinclude:: _include/envoy-dynamic-control-plane-demo.yaml + :language: yaml + :linenos: + :lines: 1-5 + :emphasize-lines: 1-3 + +.. _start_quick_start_dynamic_dynamic_resources: + +``dynamic_resources`` +--------------------- + +The :ref:`dynamic_resources ` specify +the configuration to load dynamically, and the :ref:`cluster ` +to connect to for dynamic configuration updates. + +In this example, the configuration is provided by the ``xds_cluster`` configured below. + +.. literalinclude:: _include/envoy-dynamic-control-plane-demo.yaml + :language: yaml + :linenos: + :lines: 3-19 + :lineno-start: 3 + :emphasize-lines: 3-15 + +.. _start_quick_start_dynamic_static_resources: + +``static_resources`` +-------------------- + +Here we specify the :ref:`static_resources ` +to retrieve dynamic configuration from. + +The ``xds_cluster`` is configured to query a control plane at http://my-control-plane:18000 . + +.. literalinclude:: _include/envoy-dynamic-control-plane-demo.yaml + :language: yaml + :linenos: + :lines: 17-35 + :lineno-start: 17 + :emphasize-lines: 3-17 + +.. _start_quick_start_dynamic_admin: + +``admin`` +--------- + +Configuring the :ref:`admin ` section is +the same as for :ref:`static configuration `. + +Enabling the :ref:`admin ` interface with +dynamic configuration, allows you to use the :ref:`config_dump ` +endpoint to see how Envoy is currently configured. + +.. literalinclude:: _include/envoy-dynamic-control-plane-demo.yaml + :language: yaml + :linenos: + :lines: 33-40 + :lineno-start: 33 + :emphasize-lines: 3-8 + +.. warning:: + + You may wish to restrict the network address the admin server listens to in your own deployment. diff --git a/docs/root/start/quick-start/configuration-dynamic-filesystem.rst b/docs/root/start/quick-start/configuration-dynamic-filesystem.rst new file mode 100644 index 000000000000..f6d55cf7a43b --- /dev/null +++ b/docs/root/start/quick-start/configuration-dynamic-filesystem.rst @@ -0,0 +1,115 @@ +.. _start_quick_start_dynamic_filesystem: + +Configuration: Dynamic from filesystem +====================================== + +You can start Envoy with dynamic configuration by using files that implement the :ref:`xDS ` +protocol. + +When the files are changed on the filesystem, Envoy will automatically update its configuration. + +At a minimum, you will need to start Envoy configured with the following sections: + +- :ref:`node ` to uniquely identify the proxy node. +- :ref:`dynamic_resources ` to tell Envoy where to find its + dynamic configuration. + +For the given example you will also need two dynamic configuration files: + +- :ref:`lds.yaml ` for listeners. +- :ref:`cds.yaml ` for clusters. + +You can also add an :ref:`admin ` section if you wish to monitor Envoy or +retrieve stats or configuration information. + +The following sections walk through the dynamic configuration provided in the +:download:`demo dynamic filesystem configuration file <_include/envoy-dynamic-filesystem-demo.yaml>`. + +.. _start_quick_start_dynamic_fs_node: + +``node`` +-------- + +The :ref:`node ` should specify ``cluster`` and ``id``. + +.. literalinclude:: _include/envoy-dynamic-filesystem-demo.yaml + :language: yaml + :linenos: + :lines: 1-5 + :emphasize-lines: 1-3 + +.. _start_quick_start_dynamic_fs_dynamic_resources: + +``dynamic_resources`` +--------------------- + +The :ref:`dynamic_resources ` specify +where to load dynamic configuration from. + +In this example, the configuration is provided by the ``yaml`` files set below. + +.. literalinclude:: _include/envoy-dynamic-filesystem-demo.yaml + :language: yaml + :linenos: + :lines: 3-11 + :lineno-start: 3 + :emphasize-lines: 3-7 + +.. _start_quick_start_dynamic_fs_dynamic_lds: + +``resources`` - listeners +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The linked ``lds_config`` should be an implementation of a :ref:`Listener discovery service (LDS) `. + +The following example of a :download:`dynamic LDS file <_include/envoy-dynamic-lds-demo.yaml>`, +configures an ``HTTP`` :ref:`listener ` +on port ``10000``. + +All domains and paths are matched and routed to the ``service_envoyproxy_io`` cluster. + +The ``host`` headers are rewritten to ``www.envoyproxy.io`` + +.. literalinclude:: _include/envoy-dynamic-lds-demo.yaml + :language: yaml + :linenos: + :emphasize-lines: 6-7, 20-21, 24, 26-27 + +.. _start_quick_start_dynamic_fs_dynamic_cds: + +``resources`` - clusters +~~~~~~~~~~~~~~~~~~~~~~~~ + +The linked ``cds_config`` should be an implementation of a :ref:`Cluster discovery service (CDS) `. + +In the following example of a :download:`dynamic CDS file <_include/envoy-dynamic-cds-demo.yaml>`, +the ``example_proxy_cluster`` :ref:`cluster ` +proxies over ``TLS`` to https://www.envoyproxy.io. + +.. literalinclude:: _include/envoy-dynamic-cds-demo.yaml + :language: yaml + :linenos: + :emphasize-lines: 8, 14-15, 19-20 + +.. _start_quick_start_dynamic_fs_admin: + +``admin`` +--------- + +Configuring the :ref:`admin ` section is +the same as for :ref:`static configuration `. + +Enabling the :ref:`admin ` interface with +dynamic configuration, allows you to use the :ref:`config_dump ` +endpoint to see how Envoy is currently configured. + +.. literalinclude:: _include/envoy-dynamic-filesystem-demo.yaml + :language: yaml + :linenos: + :lines: 9-16 + :lineno-start: 9 + :emphasize-lines: 3-8 + +.. warning:: + + You may wish to restrict the network address the admin server listens to in your own deployment. diff --git a/docs/root/start/quick-start/configuration-static.rst b/docs/root/start/quick-start/configuration-static.rst new file mode 100644 index 000000000000..74e539b529d0 --- /dev/null +++ b/docs/root/start/quick-start/configuration-static.rst @@ -0,0 +1,80 @@ +.. _start_quick_start_static: + +Configuration: Static +===================== + +To start Envoy with static configuration, you will need to specify :ref:`listeners ` +and :ref:`clusters ` as +:ref:`static_resources `. + +You can also add an :ref:`admin ` section if you wish to monitor Envoy +or retrieve stats. + +The following sections walk through the static configuration provided in the +:download:`demo configuration file <_include/envoy-demo.yaml>` used as the default in the Envoy Docker container. + +.. _start_quick_start_static_static_resources: + +``static_resources`` +-------------------- + +The :ref:`static_resources ` contain +everything that is configured statically when Envoy starts, as opposed to dynamically at runtime. + +.. literalinclude:: _include/envoy-demo.yaml + :language: yaml + :linenos: + :lines: 1-3 + :emphasize-lines: 1 + +.. _start_quick_start_static_listeners: + +``listeners`` +------------- + +The example configures a :ref:`listener ` +on port ``10000``. + +All paths are matched and routed to the ``service_envoyproxy_io`` +:ref:`cluster `. + +.. literalinclude:: _include/envoy-demo.yaml + :language: yaml + :linenos: + :lines: 1-29 + :emphasize-lines: 3-27 + +.. _start_quick_start_static_clusters: + +``clusters`` +------------ + +The ``service_envoyproxy_io`` :ref:`cluster ` +proxies over ``TLS`` to https://www.envoyproxy.io. + +.. literalinclude:: _include/envoy-demo.yaml + :language: yaml + :lineno-start: 27 + :lines: 27-50 + :emphasize-lines: 3-22 + +.. _start_quick_start_static_admin: + +``admin`` +--------- + +The :ref:`admin message ` is required to enable and configure +the administration server. + +The ``address`` key specifies the listening :ref:`address ` +which in the demo configuration is ``0.0.0.0:9901``. + +.. literalinclude:: _include/envoy-demo.yaml + :language: yaml + :lineno-start: 48 + :lines: 48-55 + :emphasize-lines: 3-8 + +.. warning:: + + You may wish to restrict the network address the admin server listens to in your own deployment. diff --git a/docs/root/start/quick-start/index.rst b/docs/root/start/quick-start/index.rst new file mode 100644 index 000000000000..566b82f0a084 --- /dev/null +++ b/docs/root/start/quick-start/index.rst @@ -0,0 +1,16 @@ +.. _start_quick_start: + +Quick start +=========== + +The quick start section takes you through basic operations with the Envoy server, and +provides an introduction to the types of configuration Envoy can be used with. + +.. toctree:: + :maxdepth: 3 + + run-envoy + configuration-static + configuration-dynamic-filesystem + configuration-dynamic-control-plane + next-steps diff --git a/docs/root/start/quick-start/next-steps.rst b/docs/root/start/quick-start/next-steps.rst new file mode 100644 index 000000000000..cadf47c0f6cd --- /dev/null +++ b/docs/root/start/quick-start/next-steps.rst @@ -0,0 +1,23 @@ +.. _start_quick_start_next_steps: + +Next steps +========== + +Setup a Docker deployment +------------------------- + +Learn more about :ref:`using the Envoy Docker image `. + +You can also :ref:`build your own Envoy and/or Docker image `. + +Run example sandboxes +--------------------- + +The provided :ref:`sandboxes ` are a great way to gain more experience in using Envoy, +and can be used to model your own deployments. + +Build your Envoy configuration +------------------------------ + +Check out the :ref:`configuration generator ` and other +:ref:`Envoy tools ` for working with Envoy configuration. diff --git a/docs/root/start/quick-start/run-envoy.rst b/docs/root/start/quick-start/run-envoy.rst new file mode 100644 index 000000000000..b3188c395522 --- /dev/null +++ b/docs/root/start/quick-start/run-envoy.rst @@ -0,0 +1,160 @@ +.. _start_quick_start_run_envoy: + + +Run Envoy +========= + +The following instructions walk through starting Envoy as a system daemon or using +the Envoy Docker image. + +.. _start_quick_start_version: + +Check your Envoy version +------------------------ + +Once you have :ref:`installed Envoy `, you can check the version information as follows: + +.. tabs:: + + .. tab:: System + + .. code-block:: console + + $ envoy --version + + .. tab:: Docker + + .. substitution-code-block:: console + + $ docker run --rm \ + envoyproxy/|envoy_docker_image| \ + --version + +.. _start_quick_start_help: + +View the Envoy command line options +----------------------------------- + +You can view the Envoy :ref:`command line options ` with the ``--help`` +flag: + +.. tabs:: + + .. tab:: System + + .. code-block:: console + + $ envoy --help + + .. tab:: Docker + + .. substitution-code-block:: console + + $ docker run --rm \ + envoyproxy/|envoy_docker_image| \ + --help + +.. _start_quick_start_config: + +Run Envoy with the demo configuration +------------------------------------- + +The ``-c`` or ``--config-path`` flag tells Envoy the path to its initial configuration. + +.. tabs:: + + .. tab:: System + + To start Envoy as a system daemon :download:`download the demo configuration <_include/envoy-demo.yaml>`, and start + as follows: + + .. code-block:: console + + $ envoy -c envoy-demo.yaml + + .. tab:: Docker + + You can start the Envoy Docker image without specifying a configuration file, and + it will use the demo config by default. + + .. substitution-code-block:: console + + $ docker run --rm -d \ + -p 9901:9901 \ + -p 10000:10000 \ + envoyproxy/|envoy_docker_image| + + To specify a custom configuration you can mount the config into the container, and specify the path with ``-c``. + + Assuming you have a custom configuration in the current directory named ``envoy-custom.yaml``: + + .. substitution-code-block:: console + + $ docker run --rm -d \ + -v $(pwd)/envoy-custom.yaml:/envoy-custom.yaml \ + -p 9901:9901 \ + -p 10000:10000 \ + envoyproxy/|envoy_docker_image| \ + -c /envoy-custom.yaml + +Check Envoy is proxying on http://localhost:10000 + +.. code-block:: console + + $ curl -v localhost:10000 + +The Envoy admin endpoint should also be available at http://localhost:9901 + +.. code-block:: console + + $ curl -v localhost:9901 + +.. _start_quick_start_override: + +Override the default configuration by merging a config file +----------------------------------------------------------- + +You can provide a configuration override file using ``--config-yaml`` which will merge with the main +configuration. + +Save the following snippet to ``envoy-override.yaml``: + +.. code-block:: yaml + + listeners: + - name: listener_0 + address: + socket_address: + port_value: 20000 + +Next, start the Envoy server using the override configuration. + +.. tabs:: + + .. tab:: System + + .. code-block:: console + + $ envoy -c envoy-demo.yaml --config-yaml envoy-override.yaml + + .. tab:: Docker + + .. substitution-code-block:: console + + $ docker run --rm -d \ + -v $(pwd)/envoy-override.yaml:/envoy-override.yaml \ + -p 20000:20000 \ + envoyproxy/|envoy_docker_image| \ + --config-yaml /envoy-override.yaml + +Envoy should now be proxying on http://localhost:20000 + +.. code-block:: console + + $ curl -v localhost:20000 + +The Envoy admin endpoint should also be available at http://localhost:9901 + +.. code-block:: console + + $ curl -v localhost:9901 diff --git a/docs/root/start/start.rst b/docs/root/start/start.rst index 27c3b31c1ad8..2747baa3e334 100644 --- a/docs/root/start/start.rst +++ b/docs/root/start/start.rst @@ -16,7 +16,7 @@ feature of the API, which is most useful for simple requirements. For more compl :maxdepth: 3 install - quick-start + quick-start/index docker building diff --git a/examples/dynamic-config-cp/envoy.yaml b/examples/dynamic-config-cp/envoy.yaml index 7d44f9ae6395..8f63bcfba0d6 100644 --- a/examples/dynamic-config-cp/envoy.yaml +++ b/examples/dynamic-config-cp/envoy.yaml @@ -32,15 +32,6 @@ static_resources: address: go-control-plane port_value: 18000 -layered_runtime: - layers: - - name: runtime-0 - rtds_layer: - name: runtime-0 - rtds_config: - resource_api_version: V3 - ads: {} - admin: access_log_path: /dev/null address: From 3a32d23c7c361b6ffd5860a707af8957326b2b17 Mon Sep 17 00:00:00 2001 From: Jose Ulises Nino Rivera Date: Fri, 30 Oct 2020 09:57:17 -0700 Subject: [PATCH 9/9] dns: preserve custom resolver after channel destruction (#13820) Signed-off-by: Jose Nino --- docs/root/version_history/current.rst | 1 + source/common/network/dns_impl.cc | 65 +++++++++++++---------- source/common/network/dns_impl.h | 4 ++ test/common/network/dns_impl_test.cc | 75 ++++++++++++++++++++++----- 4 files changed, 103 insertions(+), 42 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 704ca3e8b253..a33ae97a46b0 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -20,6 +20,7 @@ Bug Fixes --------- *Changes expected to improve the state of the world and are unlikely to have negative effects* +* dns: fix a bug where custom resolvers provided in configuration were not preserved after network issues. * http: fixed URL parsing for HTTP/1.1 fully qualified URLs and connect requests containing IPv6 addresses. * http: sending CONNECT_ERROR for HTTP/2 where appropriate during CONNECT requests. * tls: fix read resumption after triggering buffer high-watermark and all remaining request/response bytes are stored in the SSL connection's internal buffers. diff --git a/source/common/network/dns_impl.cc b/source/common/network/dns_impl.cc index d44d53c70f39..590aef2048f3 100644 --- a/source/common/network/dns_impl.cc +++ b/source/common/network/dns_impl.cc @@ -25,34 +25,10 @@ DnsResolverImpl::DnsResolverImpl( const bool use_tcp_for_dns_lookups) : dispatcher_(dispatcher), timer_(dispatcher.createTimer([this] { onEventCallback(ARES_SOCKET_BAD, 0); })), - use_tcp_for_dns_lookups_(use_tcp_for_dns_lookups) { - + use_tcp_for_dns_lookups_(use_tcp_for_dns_lookups), + resolvers_csv_(maybeBuildResolversCsv(resolvers)) { AresOptions options = defaultAresOptions(); initializeChannel(&options.options_, options.optmask_); - - if (!resolvers.empty()) { - std::vector resolver_addrs; - resolver_addrs.reserve(resolvers.size()); - for (const auto& resolver : resolvers) { - // This should be an IP address (i.e. not a pipe). - if (resolver->ip() == nullptr) { - ares_destroy(channel_); - throw EnvoyException( - fmt::format("DNS resolver '{}' is not an IP address", resolver->asString())); - } - // Note that the ip()->port() may be zero if the port is not fully specified by the - // Address::Instance. - // resolver->asString() is avoided as that format may be modified by custom - // Address::Instance implementations in ways that make the not a simple - // integer. See https://github.com/envoyproxy/envoy/pull/3366. - resolver_addrs.push_back(fmt::format(resolver->ip()->ipv6() ? "[{}]:{}" : "{}:{}", - resolver->ip()->addressAsString(), - resolver->ip()->port())); - } - const std::string resolvers_csv = absl::StrJoin(resolver_addrs, ","); - int result = ares_set_servers_ports_csv(channel_, resolvers_csv.c_str()); - RELEASE_ASSERT(result == ARES_SUCCESS, ""); - } } DnsResolverImpl::~DnsResolverImpl() { @@ -60,6 +36,32 @@ DnsResolverImpl::~DnsResolverImpl() { ares_destroy(channel_); } +absl::optional DnsResolverImpl::maybeBuildResolversCsv( + const std::vector& resolvers) { + if (resolvers.empty()) { + return absl::nullopt; + } + + std::vector resolver_addrs; + resolver_addrs.reserve(resolvers.size()); + for (const auto& resolver : resolvers) { + // This should be an IP address (i.e. not a pipe). + if (resolver->ip() == nullptr) { + throw EnvoyException( + fmt::format("DNS resolver '{}' is not an IP address", resolver->asString())); + } + // Note that the ip()->port() may be zero if the port is not fully specified by the + // Address::Instance. + // resolver->asString() is avoided as that format may be modified by custom + // Address::Instance implementations in ways that make the not a simple + // integer. See https://github.com/envoyproxy/envoy/pull/3366. + resolver_addrs.push_back(fmt::format(resolver->ip()->ipv6() ? "[{}]:{}" : "{}:{}", + resolver->ip()->addressAsString(), + resolver->ip()->port())); + } + return {absl::StrJoin(resolver_addrs, ",")}; +} + DnsResolverImpl::AresOptions DnsResolverImpl::defaultAresOptions() { AresOptions options{}; @@ -72,11 +74,19 @@ DnsResolverImpl::AresOptions DnsResolverImpl::defaultAresOptions() { } void DnsResolverImpl::initializeChannel(ares_options* options, int optmask) { + dirty_channel_ = false; + options->sock_state_cb = [](void* arg, os_fd_t fd, int read, int write) { static_cast(arg)->onAresSocketStateChange(fd, read, write); }; options->sock_state_cb_data = this; ares_init_options(&channel_, options, optmask | ARES_OPT_SOCK_STATE_CB); + + // Ensure that the channel points to custom resolvers, if they exist. + if (resolvers_csv_.has_value()) { + int result = ares_set_servers_ports_csv(channel_, resolvers_csv_->c_str()); + RELEASE_ASSERT(result == ARES_SUCCESS, ""); + } } void DnsResolverImpl::PendingResolution::onAresGetAddrInfoCallback(int status, int timeouts, @@ -236,12 +246,11 @@ ActiveDnsQuery* DnsResolverImpl::resolve(const std::string& dns_name, // @see DnsResolverImpl::PendingResolution::onAresGetAddrInfoCallback for why this is done. if (dirty_channel_) { - dirty_channel_ = false; ares_destroy(channel_); - AresOptions options = defaultAresOptions(); initializeChannel(&options.options_, options.optmask_); } + std::unique_ptr pending_resolution( new PendingResolution(*this, callback, dispatcher_, channel_, dns_name)); if (dns_lookup_family == DnsLookupFamily::Auto) { diff --git a/source/common/network/dns_impl.h b/source/common/network/dns_impl.h index dc62e06adb11..ace150273f31 100644 --- a/source/common/network/dns_impl.h +++ b/source/common/network/dns_impl.h @@ -87,6 +87,9 @@ class DnsResolverImpl : public DnsResolver, protected Logger::Loggable + maybeBuildResolversCsv(const std::vector& resolvers); + // Callback for events on sockets tracked in events_. void onEventCallback(os_fd_t fd, uint32_t events); // c-ares callback when a socket state changes, indicating that libevent @@ -105,6 +108,7 @@ class DnsResolverImpl : public DnsResolver, protected Logger::Loggable events_; + const absl::optional resolvers_csv_; }; } // namespace Network diff --git a/test/common/network/dns_impl_test.cc b/test/common/network/dns_impl_test.cc index 339cfb4abc89..0489bc067d39 100644 --- a/test/common/network/dns_impl_test.cc +++ b/test/common/network/dns_impl_test.cc @@ -434,18 +434,22 @@ class DnsImplTest : public testing::TestWithParam { : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")) {} void SetUp() override { - resolver_ = dispatcher_->createDnsResolver({}, use_tcp_for_dns_lookups()); - // Instantiate TestDnsServer and listen on a random port on the loopback address. server_ = std::make_unique(*dispatcher_); socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); listener_ = dispatcher_->createListener(socket_, *server_, true, ENVOY_TCP_BACKLOG_SIZE); + if (setResolverInConstructor()) { + resolver_ = dispatcher_->createDnsResolver({socket_->localAddress()}, useTcpForDnsLookups()); + } else { + resolver_ = dispatcher_->createDnsResolver({}, useTcpForDnsLookups()); + } + // Point c-ares at the listener with no search domains and TCP-only. peer_ = std::make_unique(dynamic_cast(resolver_.get())); - if (tcp_only()) { - peer_->resetChannelTcpOnly(zero_timeout()); + if (tcpOnly()) { + peer_->resetChannelTcpOnly(zeroTimeout()); } ares_set_servers_ports_csv(peer_->channel(), socket_->localAddress()->asString().c_str()); } @@ -539,9 +543,10 @@ class DnsImplTest : public testing::TestWithParam { protected: // Should the DnsResolverImpl use a zero timeout for c-ares queries? - virtual bool zero_timeout() const { return false; } - virtual bool tcp_only() const { return true; } - virtual bool use_tcp_for_dns_lookups() const { return false; } + virtual bool zeroTimeout() const { return false; } + virtual bool tcpOnly() const { return true; } + virtual bool useTcpForDnsLookups() const { return false; } + virtual bool setResolverInConstructor() const { return false; } std::unique_ptr server_; std::unique_ptr peer_; Network::MockConnectionHandler connection_handler_; @@ -579,7 +584,7 @@ TEST_P(DnsImplTest, DestructCallback) { // This simulates destruction thanks to another query setting the dirty_channel_ bit, thus causing // a subsequent result to call ares_destroy. - peer_->resetChannelTcpOnly(zero_timeout()); + peer_->resetChannelTcpOnly(zeroTimeout()); ares_set_servers_ports_csv(peer_->channel(), socket_->localAddress()->asString().c_str()); dispatcher_->run(Event::Dispatcher::RunType::Block); @@ -704,8 +709,8 @@ TEST_P(DnsImplTest, DestroyChannelOnRefused) { EXPECT_FALSE(peer_->isChannelDirty()); // Reset the channel to point to the TestDnsServer, and make sure resolution is healthy. - if (tcp_only()) { - peer_->resetChannelTcpOnly(zero_timeout()); + if (tcpOnly()) { + peer_->resetChannelTcpOnly(zeroTimeout()); } ares_set_servers_ports_csv(peer_->channel(), socket_->localAddress()->asString().c_str()); @@ -878,7 +883,7 @@ TEST_P(DnsImplTest, PendingTimerEnable) { class DnsImplZeroTimeoutTest : public DnsImplTest { protected: - bool zero_timeout() const override { return true; } + bool zeroTimeout() const override { return true; } }; // Parameterize the DNS test server socket address. @@ -898,8 +903,8 @@ TEST_P(DnsImplZeroTimeoutTest, Timeout) { class DnsImplAresFlagsForTcpTest : public DnsImplTest { protected: - bool tcp_only() const override { return false; } - bool use_tcp_for_dns_lookups() const override { return true; } + bool tcpOnly() const override { return false; } + bool useTcpForDnsLookups() const override { return true; } }; // Parameterize the DNS test server socket address. @@ -923,7 +928,7 @@ TEST_P(DnsImplAresFlagsForTcpTest, TcpLookupsEnabled) { class DnsImplAresFlagsForUdpTest : public DnsImplTest { protected: - bool tcp_only() const override { return false; } + bool tcpOnly() const override { return false; } }; // Parameterize the DNS test server socket address. @@ -945,5 +950,47 @@ TEST_P(DnsImplAresFlagsForUdpTest, UdpLookupsEnabled) { ares_destroy_options(&opts); } +class DnsImplCustomResolverTest : public DnsImplTest { + bool tcpOnly() const override { return false; } + bool useTcpForDnsLookups() const override { return true; } + bool setResolverInConstructor() const override { return true; } +}; + +// Parameterize the DNS test server socket address. +INSTANTIATE_TEST_SUITE_P(IpVersions, DnsImplCustomResolverTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(DnsImplCustomResolverTest, CustomResolverValidAfterChannelDestruction) { + ASSERT_FALSE(peer_->isChannelDirty()); + server_->addHosts("some.good.domain", {"201.134.56.7"}, RecordType::A); + server_->setRefused(true); + + EXPECT_NE(nullptr, + resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Only, + DnsResolver::ResolutionStatus::Failure, {}, {}, absl::nullopt)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + // The c-ares channel should be dirty because the TestDnsServer replied with return code REFUSED; + // This test, and the way the TestDnsServerQuery is setup, relies on the fact that Envoy's + // c-ares channel is configured **without** the ARES_FLAG_NOCHECKRESP flag. This causes c-ares to + // discard packets with REFUSED, and thus Envoy receives ARES_ECONNREFUSED due to the code here: + // https://github.com/c-ares/c-ares/blob/d7e070e7283f822b1d2787903cce3615536c5610/ares_process.c#L654 + // If that flag needs to be set, or c-ares changes its handling this test will need to be updated + // to create another condition where c-ares invokes onAresGetAddrInfoCallback with status == + // ARES_ECONNREFUSED. + EXPECT_TRUE(peer_->isChannelDirty()); + + server_->setRefused(false); + + // The next query destroys, and re-initializes the channel. Furthermore, because the test dns + // server's address was passed as a custom resolver on construction, the new channel should still + // point to the test dns server, and the query should succeed. + EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, + DnsResolver::ResolutionStatus::Success, + {"201.134.56.7"}, {}, absl::nullopt)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + EXPECT_FALSE(peer_->isChannelDirty()); +} + } // namespace Network } // namespace Envoy