admin/xDS: prepare for full /config_dump and version support (#3199)

This change does several things:
1) Clarifies how we handle xDS version_info in responses and sets us up
   for both top-level/transactional versions as well as per-resource
   versions in the future.
2) Moves the config_dump admin endpoint to the v2alpha namespace so that
   we can iterate on it in the future.
3) Fills out the config dump proto for the remaining resource types.
   These are not implemented but are here to force a discussion about
   how we want to handle versions moving forward.
4) Fixes RDS static config dump to actually work and add better tests.
5) Wire up version for the RDS config dump on a per-resource basis.

Once we agree on the general version semantics I will be following up
with dump capability of the remaining resource types.

Part of #2421
Part of #2172
Fixes #3141

Signed-off-by: Matt Klein <[email protected]>

api/envoy/admin/v2/BUILD → api/envoy/admin/v2alpha/BUILD

@@ -7,6 +7,9 @@ api_proto_library(
     srcs = ["config_dump.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//envoy/api/v2:cds",
+        "//envoy/api/v2:lds",
         "//envoy/api/v2:rds",
+        "//envoy/config/bootstrap/v2:bootstrap",
     ],
 )

api/envoy/admin/v2alpha/config_dump.proto

@@ -0,0 +1,131 @@
+syntax = "proto3";
+
+package envoy.admin.v2alpha;
+
+import "envoy/api/v2/cds.proto";
+import "envoy/api/v2/lds.proto";
+import "envoy/api/v2/rds.proto";
+import "envoy/config/bootstrap/v2/bootstrap.proto";
+
+import "google/protobuf/any.proto";
+
+import "gogoproto/gogo.proto";
+
+// [#protodoc-title: ConfigDump]
+
+// The /config_dump admin endpoint uses this wrapper message to maintain and serve arbitrary
+// configuration information from any component in Envoy.
+// TODO(jsedgwick) In the future, we may want to formalize this further with an RPC for config_dump,
+// and perhaps even with an RPC per config type. That strategy across all endpoints will allow for
+// more flexibility w.r.t. protocol, serialization, parameters, etc.
+message ConfigDump {
+  // This map is serialized and dumped in its entirety at the /config_dump endpoint.
+  //
+  // Keys should be a short descriptor of the config object they map to. For example, Envoy's HTTP
+  // routing subsystem might use "routes" as the key for its config, for which it uses the message
+  // RouteConfigDump as defined below. In the future, the key will also be used to filter the output
+  // of the /config_dump endpoint.
+  map<string, google.protobuf.Any> configs = 1 [(gogoproto.nullable) = false];
+}
+
+// This message describes the bootstrap configuration that Envoy was started with. This includes
+// any CLI overrides that were merged. Bootstrap configuration information can be used to recreate
+// the static portions of an Envoy configuration by reusing the output as the bootstrap
+// configuration for another Envoy.
+message BootstrapConfigDump {
+  envoy.config.bootstrap.v2.Bootstrap bootstrap = 1 [(gogoproto.nullable) = false];
+}
+
+// Envoy's listener manager fills this message with all currently known listeners. Listener
+// configuration information can be used to recreate an Envoy configuration by populating all
+// listeners as static listeners or by returning them in a LDS response.
+message ListenersConfigDump {
+  // This is the *version_info* in the last processed LDS discovery response. If there
+  // are only static bootstrap listeners, this field will be "".
+  string version_info = 1;
+
+  // Describes a dynamically loaded cluster via the LDS API.
+  message DynamicListener {
+    // This is the per-resource version information. This version is currently taken from the
+    // *version_info* field at the time that the listener was loaded. In the future, discrete
+    // per-listener versions may be supported by the API.
+    string version_info = 1;
+
+    // The listener config.
+    envoy.api.v2.Listener listener = 2;
+  }
+
+  // The statically loaded listener configs.
+  repeated envoy.api.v2.Listener static_listeners = 2 [(gogoproto.nullable) = false];
+
+  // The dynamically loaded active listeners. These are listeners that are available to service
+  // data plane traffic.
+  repeated DynamicListener dynamic_active_listeners = 3 [(gogoproto.nullable) = false];
+
+  // The dynamically loaded warming listeners. These are listeners that are currently undergoing
+  // warming in preparation to service data plane traffic. Note that if attempting to recreate an
+  // Envoy configuration from a configuration dump, the warming listeners should generally be
+  // discarded.
+  repeated DynamicListener dynamic_warming_listeners = 4 [(gogoproto.nullable) = false];
+
+  // The dynamically loaded draining listeners. These are listeners that are currently undergoing
+  // draining in preparation to stop servicing data plane traffic. Note that if attempting to
+  // recreate an Envoy configuration from a configuration dump, the draining listeners should
+  // generally be discarded.
+  repeated DynamicListener dynamic_draining_listeners = 5 [(gogoproto.nullable) = false];
+}
+
+// Envoy's cluster manager fills this message with all currently known clusters. Cluster
+// configuration information can be used to recreate an Envoy configuration by populating all
+// clusters as static clusters or by returning them in a CDS response.
+message ClustersConfigDump {
+  // This is the *version_info* in the last processed CDS discovery response. If there
+  // are only static bootstrap clusters, this field will be "".
+  string version_info = 1;
+
+  // Describes a dynamically loaded cluster via the CDS API.
+  message DynamicCluster {
+    // This is the per-resource version information. This version is currently taken from the
+    // *version_info* field at the time that the cluster was loaded. In the future, discrete
+    // per-cluster versions may be supported by the API.
+    string version_info = 1;
+
+    // The cluster config.
+    envoy.api.v2.Cluster cluster = 2;
+  }
+
+  // The statically loaded cluster configs.
+  repeated envoy.api.v2.Cluster static_clusters = 2 [(gogoproto.nullable) = false];
+
+  // The dynamically loaded active clusters. These are clusters that are available to service
+  // data plane traffic.
+  repeated DynamicCluster dynamic_active_clusters = 3 [(gogoproto.nullable) = false];
+
+  // The dynamically loaded warming clusters. These are clusters that are currently undergoing
+  // warming in preparation to service data plane traffic. Note that if attempting to recreate an
+  // Envoy configuration from a configuration dump, the warming clusters should generally be
+  // discarded.
+  repeated DynamicCluster dynamic_warming_clusters = 4 [(gogoproto.nullable) = false];
+}
+
+// Envoy's RDS implementation fills this message with all currently loaded routes, as described by
+// their RouteConfiguration objects. Static routes configured in the bootstrap configuration are
+// separated from those configured dynamically via RDS. Route configuration information can be used
+// to recreate an Envoy configuration by populating all routes as static routes or by returning them
+// in RDS responses.
+message RoutesConfigDump {
+  message DynamicRouteConfig {
+    // This is the per-resource version information. This version is currently taken from the
+    // *version_info* field at the time that the route configuration was loaded.
+    string version_info = 1;
+
+    // The route config.
+    envoy.api.v2.RouteConfiguration route_config = 2;
+  }
+
+  // The statically loaded route configs.
+  repeated envoy.api.v2.RouteConfiguration static_route_configs = 2 [(gogoproto.nullable) = false];
+
+  // The dynamically loaded route configs.
+  repeated DynamicRouteConfig dynamic_route_configs = 3 [(gogoproto.nullable) = false];
+}

api/envoy/config/bootstrap/v2/BUILD

@@ -5,6 +5,7 @@ licenses(["notice"])  # Apache 2
 api_proto_library(
     name = "bootstrap",
     srcs = ["bootstrap.proto"],
+    visibility = ["//visibility:public"],
     deps = [
         "//envoy/api/v2:cds",
         "//envoy/api/v2:lds",

include/envoy/config/subscription.h

@@ -21,11 +21,13 @@ template <class ResourceType> class SubscriptionCallbacks {
   /**
    * Called when a configuration update is received.
    * @param resources vector of fetched resources corresponding to the configuration update.
+   * @param version_info supplies the version information as supplied by the xDS discovery response.
    * @throw EnvoyException with reason if the configuration is rejected. Otherwise the configuration
    *        is accepted. Accepted configurations have their version_info reflected in subsequent
    *        requests.
    */
-  virtual void onConfigUpdate(const ResourceVector& resources) PURE;
+  virtual void onConfigUpdate(const ResourceVector& resources,
+                              const std::string& version_info) PURE;
 
   /**
    * Called when either the Subscription is unable to fetch a config update or when onConfigUpdate
@@ -65,16 +67,6 @@ template <class ResourceType> class Subscription {
    * @param resources vector of resource names to fetch.
    */
   virtual void updateResources(const std::vector<std::string>& resources) PURE;
-
-  /**
-   * @return std::string version info from last accepted onConfigUpdate.
-   *
-   * TODO(dnoe): This would ideally return by reference, but this causes a
-   *             problem due to incompatible string implementations returned by
-   *             protobuf generated code. Revisit when string implementations
-   *             are converged.
-   */
-  virtual const std::string versionInfo() const PURE;
 };
 
 /**

include/envoy/router/rds.h

@@ -13,31 +13,31 @@ namespace Router {
  */
 class RouteConfigProvider {
 public:
+  struct ConfigInfo {
+    // A reference to the currently loaded route configuration. Do not hold this reference beyond
+    // the caller of configInfo()'s scope.
+    const envoy::api::v2::RouteConfiguration& config_;
+
+    // The discovery version that supplied this route. This will be set to "" in the case of
+    // static clusters.
+    std::string version_;
+  };
+
   virtual ~RouteConfigProvider() {}
 
   /**
    * @return Router::ConfigConstSharedPtr a route configuration for use during a single request. The
-   * returned
-   *         config may be different on a subsequent call, so a new config should be acquired for
-   *         each request flow.
+   * returned config may be different on a subsequent call, so a new config should be acquired for
+   * each request flow.
    */
   virtual Router::ConfigConstSharedPtr config() PURE;
 
   /**
-   * @return envoy::api::v2::RouteConfiguration the underlying RouteConfiguration object associated
-   * with this provider.
-   */
-  virtual const envoy::api::v2::RouteConfiguration& configAsProto() const PURE;
-
-  /**
-   * @return const std::string version info from last accepted config.
-   *
-   * TODO(dnoe): This would ideally return by reference, but this causes a
-   *             problem due to incompatible string implementations returned by
-   *             protobuf generated code. Revisit when string implementations
-   *             are converged.
+   * @return the configuration information for the currently loaded route configuration. Note that
+   * if the provider has not yet performed an initial configuration load, no information will be
+   * returned.
    */
-  virtual const std::string versionInfo() const PURE;
+  virtual absl::optional<ConfigInfo> configInfo() const PURE;
 };
 
 typedef std::shared_ptr<RouteConfigProvider> RouteConfigProviderSharedPtr;

include/envoy/upstream/cluster_manager.h

@@ -165,14 +165,6 @@ class ClusterManager {
    */
   virtual Grpc::AsyncClientManager& grpcAsyncClientManager() PURE;
 
-  /**
-   * Return the current version info string for dynamic clusters, if CDS is setup.
-   *
-   * @return std::string the current version info string for dynamic clusters,
-   *                     or "static" if CDS is not in use.
-   */
-  virtual const std::string versionInfo() const PURE;
-
   /**
    * Return the local cluster name, if it was configured.
    *
@@ -216,11 +208,6 @@ class CdsApi {
 
   /**
    * @return std::string last accepted version from fetch.
-   *
-   * TODO(dnoe): This would ideally return by reference, but this causes a
-   *             problem due to incompatible string implementations returned by
-   *             protobuf generated code. Revisit when string implementations
-   *             are converged.
    */
   virtual const std::string versionInfo() const PURE;
 };

source/common/config/filesystem_subscription_impl.h

@@ -52,8 +52,6 @@ class FilesystemSubscriptionImpl : public Config::Subscription<ResourceType>,
     stats_.update_attempt_.inc();
   }
 
-  const std::string versionInfo() const override { return version_info_; }
-
 private:
   void refresh() {
     ENVOY_LOG(debug, "Filesystem config refresh for {}", path_);
@@ -64,13 +62,11 @@ class FilesystemSubscriptionImpl : public Config::Subscription<ResourceType>,
       MessageUtil::loadFromFile(path_, message);
       const auto typed_resources = Config::Utility::getTypedResources<ResourceType>(message);
       config_update_available = true;
-      callbacks_->onConfigUpdate(typed_resources);
-      version_info_ = message.version_info();
-      stats_.version_.set(HashUtil::xxHash64(version_info_));
+      callbacks_->onConfigUpdate(typed_resources, message.version_info());
+      stats_.version_.set(HashUtil::xxHash64(message.version_info()));
       stats_.update_success_.inc();
       ENVOY_LOG(debug, "Filesystem config update accepted for {}: {}", path_,
                 message.DebugString());
-      // TODO(htuch): Add some notion of current version for every API in stats/admin.
     } catch (const EnvoyException& e) {
       if (config_update_available) {
         ENVOY_LOG(warn, "Filesystem config update rejected: {}", e.what());
@@ -85,7 +81,6 @@ class FilesystemSubscriptionImpl : public Config::Subscription<ResourceType>,
 
   bool started_{};
   const std::string path_;
-  std::string version_info_;
   std::unique_ptr<Filesystem::Watcher> watcher_;
   SubscriptionCallbacks<ResourceType>* callbacks_{};
   SubscriptionStats stats_;

source/common/config/grpc_mux_impl.cc

@@ -195,6 +195,8 @@ void GrpcMuxImpl::onReceiveMessage(std::unique_ptr<envoy::api::v2::DiscoveryResp
       }
       watch->callbacks_.onConfigUpdate(found_resources, message->version_info());
     }
+    // TODO(mattklein123): In the future if we start tracking per-resource versions, we would do
+    // that tracking here.
     api_state_[type_url].request_.set_version_info(message->version_info());
   } catch (const EnvoyException& e) {
     ENVOY_LOG(warn, "gRPC config for {} update rejected: {}", message->type_url(), e.what());

source/common/config/grpc_mux_subscription_impl.h

@@ -41,20 +41,21 @@ class GrpcMuxSubscriptionImpl : public Subscription<ResourceType>,
     stats_.update_attempt_.inc();
   }
 
-  const std::string versionInfo() const override { return version_info_; }
-
   // Config::GrpcMuxCallbacks
   void onConfigUpdate(const Protobuf::RepeatedPtrField<ProtobufWkt::Any>& resources,
                       const std::string& version_info) override {
     Protobuf::RepeatedPtrField<ResourceType> typed_resources;
     std::transform(resources.cbegin(), resources.cend(),
                    Protobuf::RepeatedPtrFieldBackInserter(&typed_resources),
                    MessageUtil::anyConvert<ResourceType>);
-    callbacks_->onConfigUpdate(typed_resources);
+    // TODO(mattklein123): In the future if we start tracking per-resource versions, we need to
+    // supply those versions to onConfigUpdate() along with the xDS response ("system")
+    // version_info. This way, both types of versions can be tracked and exposed for debugging by
+    // the configuration update targets.
+    callbacks_->onConfigUpdate(typed_resources, version_info);
     stats_.update_success_.inc();
     stats_.update_attempt_.inc();
-    version_info_ = version_info;
-    stats_.version_.set(HashUtil::xxHash64(version_info_));
+    stats_.version_.set(HashUtil::xxHash64(version_info));
     ENVOY_LOG(debug, "gRPC config for {} accepted with {} resources: {}", type_url_,
               resources.size(), RepeatedPtrUtil::debugString(typed_resources));
   }
@@ -82,7 +83,6 @@ class GrpcMuxSubscriptionImpl : public Subscription<ResourceType>,
   const std::string type_url_;
   SubscriptionCallbacks<ResourceType>* callbacks_{};
   GrpcMuxWatchPtr watch_{};
-  std::string version_info_;
 };
 
 } // namespace Config

source/common/config/grpc_subscription_impl.h

@@ -32,8 +32,6 @@ class GrpcSubscriptionImpl : public Config::Subscription<ResourceType> {
     grpc_mux_subscription_.updateResources(resources);
   }
 
-  const std::string versionInfo() const override { return grpc_mux_subscription_.versionInfo(); }
-
   GrpcMuxImpl& grpcMux() { return grpc_mux_; }
 
 private:

source/common/config/http_subscription_impl.h

@@ -60,8 +60,6 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher,
     request_.mutable_resource_names()->Swap(&resources_vector);
   }
 
-  const std::string versionInfo() const override { return request_.version_info(); }
-
   // Http::RestApiFetcher
   void createRequest(Http::Message& request) override {
     ENVOY_LOG(debug, "Sending REST request for {}", path_);
@@ -81,7 +79,7 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher,
     }
     const auto typed_resources = Config::Utility::getTypedResources<ResourceType>(message);
     try {
-      callbacks_->onConfigUpdate(typed_resources);
+      callbacks_->onConfigUpdate(typed_resources, message.version_info());
       request_.set_version_info(message.version_info());
       stats_.version_.set(HashUtil::xxHash64(request_.version_info()));
       stats_.update_success_.inc();