Skip to content

Commit

Permalink
Move cluster dns config and validate in strict dns clusters
Browse files Browse the repository at this point in the history
Signed-off-by: Steven Jin Xuan <[email protected]>
  • Loading branch information
Stevenjin8 committed Sep 26, 2024
1 parent b203e3e commit 072cab0
Show file tree
Hide file tree
Showing 12 changed files with 218 additions and 35 deletions.
1 change: 1 addition & 0 deletions api/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ proto_library(
"//envoy/extensions/access_loggers/wasm/v3:pkg",
"//envoy/extensions/bootstrap/internal_listener/v3:pkg",
"//envoy/extensions/clusters/aggregate/v3:pkg",
"//envoy/extensions/clusters/dns/v3:pkg",
"//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg",
"//envoy/extensions/clusters/redis/v3:pkg",
"//envoy/extensions/common/async_files/v3:pkg",
Expand Down
1 change: 1 addition & 0 deletions api/envoy/config/cluster/v3/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ api_proto_package(
"//envoy/annotations:pkg",
"//envoy/config/core/v3:pkg",
"//envoy/config/endpoint/v3:pkg",
"//envoy/extensions/clusters/dns/v3:pkg",
"//envoy/type/metadata/v3:pkg",
"//envoy/type/v3:pkg",
"@com_github_cncf_xds//udpa/annotations:pkg",
Expand Down
27 changes: 7 additions & 20 deletions api/envoy/config/cluster/v3/cluster.proto
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import "envoy/config/core/v3/health_check.proto";
import "envoy/config/core/v3/protocol.proto";
import "envoy/config/core/v3/resolver.proto";
import "envoy/config/endpoint/v3/endpoint.proto";
import "envoy/extensions/clusters/dns/v3/cluster.proto";
import "envoy/type/metadata/v3/metadata.proto";
import "envoy/type/v3/percent.proto";

Expand Down Expand Up @@ -45,7 +46,7 @@ message ClusterCollection {
}

// Configuration for a single upstream cluster.
// [#next-free-field: 59]
// [#next-free-field: 60]
message Cluster {
option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Cluster";

Expand Down Expand Up @@ -678,24 +679,6 @@ message Cluster {
core.v3.HealthStatusSet override_host_status = 8;
}

message RefreshRate {
option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Cluster.RefreshRate";

// Specifies the base interval between refreshes. This parameter is required and must be greater
// than zero and less than
// :ref:`max_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.max_interval>`.
google.protobuf.Duration base_interval = 1 [(validate.rules).duration = {
required: true
gt {nanos: 1000000}
}];

// Specifies the maximum interval between refreshes. This parameter is optional, but must be
// greater than or equal to the
// :ref:`base_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.base_interval>` if set. The default
// is 10 times the :ref:`base_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.base_interval>`.
google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}];
}

message PreconnectPolicy {
// Indicates how many streams (rounded up) can be anticipated per-upstream for each
// incoming stream. This is useful for high-QPS or latency-sensitive services. Preconnecting
Expand Down Expand Up @@ -944,6 +927,9 @@ message Cluster {
// [#next-major-version: make this a list of typed extensions.]
map<string, google.protobuf.Any> typed_extension_protocol_options = 36;

// [#extension-category: envoy.clusters.dns]
envoy.extensions.clusters.dns.v3.DnsConfig dns_config = 59;

// If the DNS refresh rate is specified and the cluster type is either
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>`,
// or :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`,
Expand Down Expand Up @@ -975,7 +961,8 @@ message Cluster {
// other than :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>` and
// :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>` this setting is
// ignored.
RefreshRate dns_failure_refresh_rate = 44;

envoy.extensions.clusters.dns.v3.DnsConfig.RefreshRate dns_failure_refresh_rate = 44;

// Optional configuration for setting cluster's DNS refresh rate. If the value is set to true,
// cluster's DNS refresh rate will be set to resource record's TTL which comes from DNS
Expand Down
9 changes: 9 additions & 0 deletions api/envoy/extensions/clusters/dns/v3/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.

load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")

licenses(["notice"]) # Apache 2

api_proto_package(
deps = ["@com_github_cncf_xds//udpa/annotations:pkg"],
)
130 changes: 130 additions & 0 deletions api/envoy/extensions/clusters/dns/v3/cluster.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
syntax = "proto3";

package envoy.extensions.clusters.dns.v3;

import "google/protobuf/any.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/wrappers.proto";

import "udpa/annotations/migrate.proto";
import "udpa/annotations/security.proto";
import "udpa/annotations/status.proto";
import "udpa/annotations/versioning.proto";
import "validate/validate.proto";

option java_package = "io.envoyproxy.envoy.extensions.clusters.dns.v3";
option java_outer_classname = "ClusterProto";
option java_multiple_files = true;
option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dns/v3;dnsv3";
option (udpa.annotations.file_status).package_version_status = ACTIVE;

// import "envoy/config/core/v3/extension.proto";

// [#protodoc-title: DNS cluster configuration]

// Configuration for the dynamic forward proxy cluster. See the :ref:`architecture overview
// <arch_overview_http_dynamic_forward_proxy>` for more information.
// [#extension: envoy.clusters.dns]

// [#next-free-field: 55]
message DnsConfig {
// When V4_ONLY is selected, the DNS resolver will only perform a lookup for
// addresses in the IPv4 family. If V6_ONLY is selected, the DNS resolver will
// only perform a lookup for addresses in the IPv6 family. If AUTO is
// specified, the DNS resolver will first perform a lookup for addresses in
// the IPv6 family and fallback to a lookup for addresses in the IPv4 family.
// This is semantically equivalent to a non-existent V6_PREFERRED option.
// AUTO is a legacy name that is more opaque than
// necessary and will be deprecated in favor of V6_PREFERRED in a future major version of the API.
// If V4_PREFERRED is specified, the DNS resolver will first perform a lookup for addresses in the
// IPv4 family and fallback to a lookup for addresses in the IPv6 family. i.e., the callback
// target will only get v6 addresses if there were NO v4 addresses to return.
// If ALL is specified, the DNS resolver will perform a lookup for both IPv4 and IPv6 families,
// and return all resolved addresses. When this is used, Happy Eyeballs will be enabled for
// upstream connections. Refer to :ref:`Happy Eyeballs Support <arch_overview_happy_eyeballs>`
// for more information.
// For cluster types other than
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>` and
// :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`,
// this setting is
// ignored.
// [#next-major-version: deprecate AUTO in favor of a V6_PREFERRED option.]
enum LookupFamily {
AUTO = 0;
V4_ONLY = 1;
V6_ONLY = 2;
V4_PREFERRED = 3;
ALL = 4;
}

message RefreshRate {
option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Cluster.RefreshRate";

// Specifies the base interval between refreshes. This parameter is required and must be greater
// than zero and less than
// :ref:`max_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.max_interval>`.
google.protobuf.Duration base_interval = 1 [(validate.rules).duration = {
required: true
gt {nanos: 1000000}
}];

// Specifies the maximum interval between refreshes. This parameter is optional, but must be
// greater than or equal to the
// :ref:`base_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.base_interval>` if set. The default
// is 10 times the :ref:`base_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.base_interval>`.
google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}];
}

// If the DNS refresh rate is specified and the cluster type is either
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>`,
// or :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`,
// this value is used as the cluster’s DNS refresh
// rate. The value configured must be at least 1ms. If this setting is not specified, the
// value defaults to 5000ms. For cluster types other than
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>`
// and :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`
// this setting is ignored.
google.protobuf.Duration refresh_rate = 16 [(validate.rules).duration = {gt {nanos: 1000000}}];

// If the DNS failure refresh rate is specified and the cluster type is either
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>`,
// or :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`,
// this is used as the cluster’s DNS refresh rate when requests are failing. If this setting is
// not specified, the failure refresh rate defaults to the DNS refresh rate. For cluster types
// other than :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>` and
// :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>` this setting is
// ignored.
RefreshRate dns_failure_refresh_rate = 44;

// Optional configuration for setting cluster's DNS refresh rate. If the value is set to true,
// cluster's DNS refresh rate will be set to resource record's TTL which comes from DNS
// resolution.
bool respect_ttl = 39;

// The DNS IP address resolution policy. If this setting is not specified, the
// value defaults to
// :ref:`AUTO<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DnsLookupFamily.AUTO>`.
LookupFamily lookup_family = 17 [(validate.rules).enum = {defined_only: true}];

// DNS resolver type configuration extension. This extension can be used to configure c-ares, apple,
// or any other DNS resolver types and the related parameters.
// For example, an object of
// :ref:`CaresDnsResolverConfig <envoy_v3_api_msg_extensions.network.dns_resolver.cares.v3.CaresDnsResolverConfig>`
// can be packed into this ``typed_dns_resolver_config``. This configuration replaces the
// :ref:`dns_resolution_config <envoy_v3_api_field_config.cluster.v3.Cluster.dns_resolution_config>`
// configuration.
// During the transition period when both ``dns_resolution_config`` and ``typed_dns_resolver_config`` exists,
// when ``typed_dns_resolver_config`` is in place, Envoy will use it and ignore ``dns_resolution_config``.
// When ``typed_dns_resolver_config`` is missing, the default behavior is in place.
// [#extension-category: envoy.network.dns_resolver]
// FIXMEcore.v3.TypedExtensionConfig typed_resolver_config = 55;

// Optional configuration for having cluster readiness block on warm-up. Currently, only applicable for
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>`,
// or :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`,
// or :ref:`Redis Cluster<arch_overview_redis>`.
// If true, cluster readiness blocks on warm-up. If false, the cluster will complete
// initialization whether or not warm-up has completed. Defaults to true.
google.protobuf.BoolValue wait_for_warm_on_init = 54;
}
1 change: 1 addition & 0 deletions api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ api_proto_package(
"//envoy/config/cluster/v3:pkg",
"//envoy/config/common/key_value/v3:pkg",
"//envoy/config/core/v3:pkg",
"//envoy/extensions/clusters/dns/v3:pkg",
"@com_github_cncf_xds//udpa/annotations:pkg",
],
)
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import "envoy/config/common/key_value/v3/config.proto";
import "envoy/config/core/v3/address.proto";
import "envoy/config/core/v3/extension.proto";
import "envoy/config/core/v3/resolver.proto";
import "envoy/extensions/clusters/dns/v3/cluster.proto";

import "google/protobuf/duration.proto";
import "google/protobuf/wrappers.proto";
Expand Down Expand Up @@ -97,7 +98,7 @@ message DnsCacheConfig {
// If the DNS failure refresh rate is specified,
// this is used as the cache's DNS refresh rate when DNS requests are failing. If this setting is
// not specified, the failure refresh rate defaults to the dns_refresh_rate.
config.cluster.v3.Cluster.RefreshRate dns_failure_refresh_rate = 6;
clusters.dns.v3.DnsConfig.RefreshRate dns_failure_refresh_rate = 6;

// The config of circuit breakers for resolver. It provides a configurable threshold.
// Envoy will use dns cache circuit breakers with default settings even if this value is not set.
Expand Down
1 change: 1 addition & 0 deletions api/versioning/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ proto_library(
"//envoy/extensions/access_loggers/wasm/v3:pkg",
"//envoy/extensions/bootstrap/internal_listener/v3:pkg",
"//envoy/extensions/clusters/aggregate/v3:pkg",
"//envoy/extensions/clusters/dns/v3:pkg",
"//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg",
"//envoy/extensions/clusters/redis/v3:pkg",
"//envoy/extensions/common/async_files/v3:pkg",
Expand Down
1 change: 1 addition & 0 deletions source/extensions/clusters/strict_dns/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ envoy_cc_extension(
# prevously considered core code.
visibility = ["//visibility:public"],
deps = [
"//source/common/common:random_generator_lib",
"//source/common/upstream:cluster_factory_includes",
"//source/common/upstream:upstream_includes",
"@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
Expand Down
61 changes: 51 additions & 10 deletions source/extensions/clusters/strict_dns/strict_dns_cluster.cc
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
#include "source/extensions/clusters/strict_dns/strict_dns_cluster.h"

#include <chrono>
#include <random>

#include "envoy/common/exception.h"
#include "envoy/config/cluster/v3/cluster.pb.h"
#include "envoy/config/endpoint/v3/endpoint.pb.h"
#include "envoy/config/endpoint/v3/endpoint_components.pb.h"

#include "source/common/common/random_generator.h"

namespace Envoy {
namespace Upstream {

Expand All @@ -28,11 +31,43 @@ StrictDnsClusterImpl::StrictDnsClusterImpl(const envoy::config::cluster::v3::Clu
absl::Status& creation_status)
: BaseDynamicClusterImpl(cluster, context, creation_status),
load_assignment_(cluster.load_assignment()),
local_info_(context.serverFactoryContext().localInfo()), dns_resolver_(dns_resolver),
dns_refresh_rate_ms_(
std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_refresh_rate, 5000))),
dns_jitter_ms_(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_jitter, 0)),
respect_dns_ttl_(cluster.respect_dns_ttl()) {
local_info_(context.serverFactoryContext().localInfo()), dns_resolver_(dns_resolver) {
// dns_refresh_rate_ms_(
// std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_refresh_rate,
// 5000))),
// dns_jitter_ms_(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_jitter, 0)),
// respect_dns_ttl_(cluster.respect_dns_ttl())

if (cluster.has_dns_config()) {
if (cluster.has_dns_refresh_rate() /* deprecated */) {
throw EnvoyException("Only one of dns_refresh_rate or dns_config can be specified.");
}
if (cluster.has_dns_jitter() /* deprecated */) {
throw EnvoyException("Only one of dns_jitter or dns_config can be specified.");
}
if (cluster.has_typed_dns_resolver_config() /* deprecated */) {
throw EnvoyException(
"Only one of typed_dns_resolution_config or dns_config can be specified.");
}
if (cluster.dns_lookup_family() !=
envoy::config::cluster::v3::Cluster_DnsLookupFamily::Cluster_DnsLookupFamily_AUTO &&
cluster.dns_config().lookup_family() !=
envoy::extensions::clusters::dns::v3::DnsConfig::LookupFamily::
DnsConfig_LookupFamily_AUTO /* deprecated */) {
throw EnvoyException("Only one of dns_lookup_family or dns_config can be specified.");
}
if (cluster.has_dns_refresh_rate() /* deprecated */) {
throw EnvoyException("Only one of dns_refresh_rate or dns_config can be specified.");
}
if (cluster.has_dns_failure_refresh_rate() /* deprecated */) {
throw EnvoyException("Only one of dns_failure_refresh_rate or dns_config can be specified.");
}

if (cluster.has_wait_for_warm_on_init() /* deprecated */) {
throw EnvoyException("Only one of wait_for_warm_on_init or dns_config can be specified.");
}
// no good way of checking respect_ttl
}
failure_backoff_strategy_ =
Config::Utility::prepareDnsRefreshStrategy<envoy::config::cluster::v3::Cluster>(
cluster, dns_refresh_rate_ms_.count(),
Expand All @@ -51,7 +86,7 @@ StrictDnsClusterImpl::StrictDnsClusterImpl(const envoy::config::cluster::v3::Clu

resolve_targets.emplace_back(new ResolveTarget(
*this, context.serverFactoryContext().mainThreadDispatcher(), socket_address.address(),
socket_address.port_value(), locality_lb_endpoint, lb_endpoint));
socket_address.port_value(), locality_lb_endpoint, lb_endpoint, random_generator_));
}
}
resolve_targets_ = std::move(resolve_targets);
Expand Down Expand Up @@ -103,13 +138,14 @@ StrictDnsClusterImpl::ResolveTarget::ResolveTarget(
StrictDnsClusterImpl& parent, Event::Dispatcher& dispatcher, const std::string& dns_address,
const uint32_t dns_port,
const envoy::config::endpoint::v3::LocalityLbEndpoints& locality_lb_endpoint,
const envoy::config::endpoint::v3::LbEndpoint& lb_endpoint)
const envoy::config::endpoint::v3::LbEndpoint& lb_endpoint,
Random::RandomGeneratorImpl& random_generator)
: parent_(parent), locality_lb_endpoints_(locality_lb_endpoint), lb_endpoint_(lb_endpoint),
dns_address_(dns_address),
hostname_(lb_endpoint_.endpoint().hostname().empty() ? dns_address_
: lb_endpoint_.endpoint().hostname()),
port_(dns_port),
resolve_timer_(dispatcher.createTimer([this]() -> void { startResolve(); })) {}
port_(dns_port), resolve_timer_(dispatcher.createTimer([this]() -> void { startResolve(); })),
random_generator_(random_generator) {}

StrictDnsClusterImpl::ResolveTarget::~ResolveTarget() {
if (active_query_) {
Expand Down Expand Up @@ -188,7 +224,12 @@ void StrictDnsClusterImpl::ResolveTarget::startResolve() {

if (!response.empty() && parent_.respect_dns_ttl_ &&
ttl_refresh_rate != std::chrono::seconds(0)) {
final_refresh_rate = ttl_refresh_rate;
std::chrono::milliseconds jitter =
std::chrono::milliseconds(random_generator_.random() % 512); // FIXME
if (jitter >= ttl_refresh_rate) {
jitter = std::chrono::milliseconds(0);
}
final_refresh_rate = ttl_refresh_rate - jitter;
ASSERT(ttl_refresh_rate != std::chrono::seconds::max() &&
final_refresh_rate.count() > 0);
}
Expand Down
Loading

0 comments on commit 072cab0

Please sign in to comment.