From 57182872200547b6b3c6af1b2feb51b094f4413c Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 12 Dec 2022 11:31:30 +0000 Subject: [PATCH 01/11] Chunked encoding for cluster state API Relates #89838 --- .../reroute/ClusterRerouteResponse.java | 4 +- .../elasticsearch/cluster/ClusterState.java | 231 +++++++++--------- .../xcontent/ChunkedToXContentHelper.java | 4 + .../DispatchingRestToXContentListener.java | 53 ---- .../action/RestChunkedToXContentListener.java | 10 +- .../admin/cluster/RestClusterStateAction.java | 53 ++-- .../cluster/ClusterStateTests.java | 24 +- .../elasticsearch/test/XContentTestUtils.java | 5 + .../cluster/ClusterStatsMonitoringDoc.java | 5 +- 9 files changed, 191 insertions(+), 198 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/rest/action/DispatchingRestToXContentListener.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index 4e4743aea5581..56394e435ac9c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.xcontent.ToXContentObject; @@ -75,7 +76,8 @@ protected void addCustomFields(XContentBuilder builder, Params params) throws IO deprecationLogger.critical(DeprecationCategory.API, "reroute_cluster_state", STATE_FIELD_DEPRECATION_MESSAGE); } builder.startObject("state"); - state.toXContent(builder, params); + // TODO this should be chunked, see #89838 + ChunkedToXContent.wrapAsXContentObject(state).toXContent(builder, params); builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index ccae40d38ddef..aa7cb0184e04a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -21,12 +21,9 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterApplierService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterService; @@ -35,6 +32,7 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -44,13 +42,13 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContent; -import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; @@ -59,6 +57,7 @@ import java.util.Set; import java.util.concurrent.Executor; import java.util.function.Consumer; +import java.util.function.Function; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; @@ -95,7 +94,7 @@ *

* Cluster state updates can be used to trigger various actions via a {@link ClusterStateListener} rather than using a timer. *

- * Implements {@link ToXContentFragment} to be exposed in REST APIs (e.g. {@code GET _cluster/state} and {@code POST _cluster/reroute}) and + * Implements {@link ChunkedToXContent} to be exposed in REST APIs (e.g. {@code GET _cluster/state} and {@code POST _cluster/reroute}) and * to be indexed by monitoring, mostly just for diagnostics purposes. The {@link XContent} representation does not need to be 100% faithful * since we never reconstruct a cluster state from its XContent representation, but the more faithful it is the more useful it is for * diagnostics. Note that the {@link XContent} representation of the {@link Metadata} portion does have to be faithful (in {@link @@ -104,7 +103,7 @@ * Security-sensitive data such as passwords or private keys should not be stored in the cluster state, since the contents of the cluster * state are exposed in various APIs. */ -public class ClusterState implements ToXContentFragment, Diffable { +public class ClusterState implements ChunkedToXContent, Diffable { public static final ClusterState EMPTY_STATE = builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build(); @@ -124,7 +123,7 @@ default boolean isPrivate() { * the more faithful it is the more useful it is for diagnostics. */ @Override - Iterator toXContentChunked(Params params); + Iterator toXContentChunked(ToXContent.Params params); } private static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); @@ -520,114 +519,128 @@ public String toString() { } } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - EnumSet metrics = Metric.parseString(params.param("metric", "_all"), true); - - // always provide the cluster_uuid as part of the top-level response (also part of the metadata response) - builder.field("cluster_uuid", metadata().clusterUUID()); - - if (metrics.contains(Metric.VERSION)) { - builder.field("version", version); - builder.field("state_uuid", stateUUID); - } - - if (metrics.contains(Metric.MASTER_NODE)) { - builder.field("master_node", nodes().getMasterNodeId()); - } - - if (metrics.contains(Metric.BLOCKS)) { - builder.startObject("blocks"); - - if (blocks().global().isEmpty() == false) { - builder.startObject("global"); - for (ClusterBlock block : blocks().global()) { - block.toXContent(builder, params); - } - builder.endObject(); - } + private static Iterator chunkedSection( + boolean condition, + ToXContent before, + Iterator items, + Function> fn, + ToXContent after + ) { + return condition + ? Iterators.concat(Iterators.single(before), Iterators.flatMap(items, fn::apply), Iterators.single(after)) + : Collections.emptyIterator(); + } - if (blocks().indices().isEmpty() == false) { - builder.startObject("indices"); - for (Map.Entry> entry : blocks().indices().entrySet()) { - builder.startObject(entry.getKey()); - for (ClusterBlock block : entry.getValue()) { + @Override + public Iterator toXContentChunked(ToXContent.Params outerParams) { + final var metrics = Metric.parseString(outerParams.param("metric", "_all"), true); + + return Iterators.concat( + // always provide the cluster_uuid as part of the top-level response (also part of the metadata response) + ChunkedToXContentHelper.field("cluster_uuid", metadata().clusterUUID()), + + // state version info + metrics.contains(Metric.VERSION) + ? Iterators.single((builder2, params2) -> builder2.field("version", version).field("state_uuid", stateUUID)) + : Collections.emptyIterator(), + + // master node + metrics.contains(Metric.MASTER_NODE) + ? Iterators.single((builder1, params1) -> builder1.field("master_node", nodes().getMasterNodeId())) + : Collections.emptyIterator(), + + // blocks + chunkedSection(metrics.contains(Metric.BLOCKS), (builder, params) -> { + builder.startObject("blocks"); + if (blocks().global().isEmpty() == false) { + builder.startObject("global"); + for (ClusterBlock block : blocks().global()) { block.toXContent(builder, params); } builder.endObject(); } - builder.endObject(); - } - - builder.endObject(); - } - - // nodes - if (metrics.contains(Metric.NODES)) { - builder.startObject("nodes"); - for (DiscoveryNode node : nodes) { - node.toXContent(builder, params); - } - builder.endObject(); - } - - // meta data - if (metrics.contains(Metric.METADATA)) { - ChunkedToXContent.wrapAsXContentObject(metadata).toXContent(builder, params); - } - - // routing table - if (metrics.contains(Metric.ROUTING_TABLE)) { - builder.startObject("routing_table"); - builder.startObject("indices"); - for (IndexRoutingTable indexRoutingTable : routingTable()) { - builder.startObject(indexRoutingTable.getIndex().getName()); - builder.startObject("shards"); - for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) { - IndexShardRoutingTable indexShardRoutingTable = indexRoutingTable.shard(shardId); - builder.startArray(Integer.toString(indexShardRoutingTable.shardId().id())); - for (int copy = 0; copy < indexShardRoutingTable.size(); copy++) { - indexShardRoutingTable.shard(copy).toXContent(builder, params); - } - builder.endArray(); + if (blocks().indices().isEmpty() == false) { + builder.startObject("indices"); } - builder.endObject(); - builder.endObject(); - } - builder.endObject(); - builder.endObject(); - } - - // routing nodes - if (metrics.contains(Metric.ROUTING_NODES)) { - builder.startObject("routing_nodes"); - builder.startArray("unassigned"); - for (ShardRouting shardRouting : getRoutingNodes().unassigned()) { - shardRouting.toXContent(builder, params); - } - builder.endArray(); - - builder.startObject("nodes"); - for (RoutingNode routingNode : getRoutingNodes()) { - builder.startArray(routingNode.nodeId() == null ? "null" : routingNode.nodeId()); - for (ShardRouting shardRouting : routingNode) { - shardRouting.toXContent(builder, params); + return builder; + }, blocks.indices().entrySet().iterator(), entry -> Iterators.single((builder, params) -> { + builder.startObject(entry.getKey()); + for (ClusterBlock block : entry.getValue()) { + block.toXContent(builder, params); } - builder.endArray(); - } - builder.endObject(); - - builder.endObject(); - } - if (metrics.contains(Metric.CUSTOMS)) { - for (Map.Entry cursor : customs.entrySet()) { - builder.startObject(cursor.getKey()); - ChunkedToXContent.wrapAsXContentObject(cursor.getValue()).toXContent(builder, params); - builder.endObject(); - } - } - - return builder; + return builder.endObject(); + }), (builder, params) -> { + if (blocks().indices().isEmpty() == false) { + builder.endObject(); + } + return builder.endObject(); + }), + + // nodes + chunkedSection( + metrics.contains(Metric.NODES), + (builder, params) -> builder.startObject("nodes"), + nodes.iterator(), + Iterators::single, + (builder, params) -> builder.endObject() + ), + + // metadata + metrics.contains(Metric.METADATA) ? metadata.toXContentChunked(outerParams) : Collections.emptyIterator(), + + // routing table + chunkedSection( + metrics.contains(Metric.ROUTING_TABLE), + (builder, params) -> builder.startObject("routing_table").startObject("indices"), + routingTable().iterator(), + indexRoutingTable -> Iterators.single((builder, params) -> { + builder.startObject(indexRoutingTable.getIndex().getName()); + builder.startObject("shards"); + for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) { + IndexShardRoutingTable indexShardRoutingTable = indexRoutingTable.shard(shardId); + builder.startArray(Integer.toString(indexShardRoutingTable.shardId().id())); + for (int copy = 0; copy < indexShardRoutingTable.size(); copy++) { + indexShardRoutingTable.shard(copy).toXContent(builder, params); + } + builder.endArray(); + } + return builder.endObject().endObject(); + }), + (builder, params) -> builder.endObject().endObject() + ), + + // routing nodes + chunkedSection( + metrics.contains(Metric.ROUTING_NODES), + (builder, params) -> builder.startObject("routing_nodes").startArray("unassigned"), + getRoutingNodes().unassigned().iterator(), + Iterators::single, + (builder, params) -> builder.endArray() // no endObject() here, continued in next chunkedSection() + ), + chunkedSection( + metrics.contains(Metric.ROUTING_NODES), + (builder, params) -> builder.startObject("nodes"), + getRoutingNodes().iterator(), + routingNode -> Iterators.concat( + ChunkedToXContentHelper.startArray(routingNode.nodeId() == null ? "null" : routingNode.nodeId()), + routingNode.iterator(), + ChunkedToXContentHelper.endArray() + ), + (builder, params) -> builder.endObject().endObject() + ), + + // customs + metrics.contains(Metric.CUSTOMS) + ? Iterators.flatMap( + customs.entrySet().iterator(), + cursor -> Iterators.concat( + ChunkedToXContentHelper.startObject(cursor.getKey()), + cursor.getValue().toXContentChunked(outerParams), + ChunkedToXContentHelper.endObject() + ) + ) + : Collections.emptyIterator() + ); } public static Builder builder(ClusterName clusterName) { diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java index 19ebbe350a53f..4a52024f137fc 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java @@ -62,6 +62,10 @@ public static Iterator field(String name, boolean value) { return Iterators.single(((builder, params) -> builder.field(name, value))); } + public static Iterator field(String name, String value) { + return Iterators.single(((builder, params) -> builder.field(name, value))); + } + public static Iterator array(String name, Iterator contents) { return Iterators.concat(ChunkedToXContentHelper.startArray(name), contents, ChunkedToXContentHelper.endArray()); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/DispatchingRestToXContentListener.java b/server/src/main/java/org/elasticsearch/rest/action/DispatchingRestToXContentListener.java deleted file mode 100644 index be4cbd120dcb5..0000000000000 --- a/server/src/main/java/org/elasticsearch/rest/action/DispatchingRestToXContentListener.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.rest.action; - -import org.elasticsearch.action.ActionRunnable; -import org.elasticsearch.rest.RestChannel; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.util.concurrent.ExecutorService; - -/** - * Response listener for REST requests which dispatches the serialization of the response off of the thread on which the response was - * received, since that thread is often a transport thread and XContent serialization might be expensive. - */ -public class DispatchingRestToXContentListener extends RestActionListener { - - private final ExecutorService executor; - private final RestRequest restRequest; - - public DispatchingRestToXContentListener(ExecutorService executor, RestChannel channel, RestRequest restRequest) { - super(channel); - this.executor = executor; - this.restRequest = restRequest; - } - - protected ToXContent.Params getParams() { - return restRequest; - } - - @Override - protected void processResponse(Response response) { - executor.execute(ActionRunnable.wrap(this, l -> new RestBuilderListener(channel) { - @Override - public RestResponse buildResponse(final Response response, final XContentBuilder builder) throws Exception { - ensureOpen(); - response.toXContent(builder, getParams()); - return new RestResponse(RestStatus.OK, builder); - } - }.onResponse(response))); - } - -} diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java b/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java index ece3a432f059b..27e2b70feecfe 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java +++ b/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java @@ -13,6 +13,7 @@ import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; @@ -22,12 +23,19 @@ */ public final class RestChunkedToXContentListener extends RestActionListener { + private final ToXContent.Params params; + public RestChunkedToXContentListener(RestChannel channel) { + this(channel, channel.request()); + } + + public RestChunkedToXContentListener(RestChannel channel, ToXContent.Params params) { super(channel); + this.params = params; } @Override protected void processResponse(Response response) throws IOException { - channel.sendResponse(new RestResponse(RestStatus.OK, ChunkedRestResponseBody.fromXContent(response, channel.request(), channel))); + channel.sendResponse(new RestResponse(RestStatus.OK, ChunkedRestResponseBody.fromXContent(response, params, channel))); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java index c3a79c1cc42ef..7928f35049302 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java @@ -18,19 +18,22 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.DispatchingRestToXContentListener; import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestChunkedToXContentListener; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collections; import java.util.EnumSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.function.LongSupplier; @@ -108,21 +111,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( ClusterStateAction.INSTANCE, clusterStateRequest, - new DispatchingRestToXContentListener( - // Process serialization on MANAGEMENT pool since the serialization of the cluster state to XContent - // can be too slow to execute on an IO thread - threadPool.executor(ThreadPool.Names.MANAGEMENT), + new RestChunkedToXContentListener( channel, - request - ) { - @Override - protected ToXContent.Params getParams() { - return new ToXContent.DelegatingMapParams( - singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API), - request - ); - } - }.map(response -> new RestClusterStateResponse(clusterStateRequest, response, threadPool::relativeTimeInMillis)) + new ToXContent.DelegatingMapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API), request) + ).map(response -> new RestClusterStateResponse(clusterStateRequest, response, threadPool::relativeTimeInMillis)) ); } @@ -141,7 +133,7 @@ static final class Fields { static final String CLUSTER_NAME = "cluster_name"; } - private static class RestClusterStateResponse implements ToXContentObject { + private static class RestClusterStateResponse implements ChunkedToXContent { private final ClusterStateRequest request; private final ClusterStateResponse response; @@ -156,22 +148,25 @@ private static class RestClusterStateResponse implements ToXContentObject { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + public Iterator toXContentChunked(ToXContent.Params outerParams) { if (request.local() == false && currentTimeMillisSupplier.getAsLong() - startTimeMillis > request.masterNodeTimeout().millis()) { throw new ElasticsearchTimeoutException("Timed out getting cluster state"); } - builder.startObject(); - if (request.waitForMetadataVersion() != null) { - builder.field(Fields.WAIT_FOR_TIMED_OUT, response.isWaitForTimedOut()); - } - builder.field(Fields.CLUSTER_NAME, response.getClusterName().value()); + final ClusterState responseState = response.getState(); - if (responseState != null) { - responseState.toXContent(builder, params); - } - builder.endObject(); - return builder; + + return Iterators.concat(Iterators.single((builder, params) -> { + builder.startObject(); + if (request.waitForMetadataVersion() != null) { + builder.field(Fields.WAIT_FOR_TIMED_OUT, response.isWaitForTimedOut()); + } + builder.field(Fields.CLUSTER_NAME, response.getClusterName().value()); + return builder; + }), + responseState == null ? Collections.emptyIterator() : responseState.toXContentChunked(outerParams), + ChunkedToXContentHelper.endObject() + ); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 79dfd0be6b238..cc461ac44b1ab 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -138,7 +138,12 @@ public void testToXContent() throws IOException { XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); - clusterState.toXContent(builder, new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API))); + writeChunks( + clusterState, + builder, + new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API)), + 36 + ); builder.endObject(); assertEquals(XContentHelper.stripWhitespace(formatted(""" @@ -369,7 +374,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); builder.startObject(); - clusterState.toXContent(builder, new ToXContent.MapParams(mapParams)); + writeChunks(clusterState, builder, new ToXContent.MapParams(mapParams), 36); builder.endObject(); assertEquals(formatted(""" @@ -597,7 +602,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); builder.startObject(); - clusterState.toXContent(builder, new ToXContent.MapParams(mapParams)); + writeChunks(clusterState, builder, new ToXContent.MapParams(mapParams), 36); builder.endObject(); assertEquals(formatted(""" @@ -849,7 +854,7 @@ public void testToXContentSameTypeName() throws IOException { XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); builder.startObject(); - clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS); + writeChunks(clusterState, builder, ToXContent.EMPTY_PARAMS, 27); builder.endObject(); assertEquals(formatted(""" @@ -1040,4 +1045,15 @@ public void testNodesIfRecovered() throws IOException { .build(); assertEquals(DiscoveryNodes.EMPTY_NODES, notRecoveredState.nodesIfRecovered()); } + + private static void writeChunks(ClusterState clusterState, XContentBuilder builder, ToXContent.Params params, int expectedChunks) + throws IOException { + final var iterator = clusterState.toXContentChunked(params); + int chunks = 0; + while (iterator.hasNext()) { + iterator.next().toXContent(builder, params); + chunks += 1; + } + assertEquals(expectedChunks, chunks); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java index bcc4c64f015f3..fcc0bd7021dd7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java @@ -9,6 +9,7 @@ package org.elasticsearch.test; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.xcontent.DeprecationHandler; @@ -41,6 +42,10 @@ private XContentTestUtils() { } + public static Map convertToMap(ChunkedToXContent chunkedToXContent) throws IOException { + return convertToMap(ChunkedToXContent.wrapAsXContentObject(chunkedToXContent)); + } + public static Map convertToMap(ToXContent part) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); if (part.isFragment()) { diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java index d669f069efd8b..1ef3f7301c54d 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.core.Nullable; import org.elasticsearch.license.License; import org.elasticsearch.xcontent.ToXContent; @@ -162,7 +163,9 @@ protected void innerToXContent(XContentBuilder builder, Params params) throws IO { builder.field("nodes_hash", nodesHash(clusterState.nodes())); builder.field("status", status.name().toLowerCase(Locale.ROOT)); - clusterState.toXContent(builder, CLUSTER_STATS_PARAMS); + // we need the whole doc in memory anyway so no need to preserve chunking here; moreover CLUSTER_STATS_PARAMS doesn't + // include anything heavy so this should be fine. + ChunkedToXContent.wrapAsXContentObject(clusterState).toXContent(builder, CLUSTER_STATS_PARAMS); } builder.endObject(); } From c90afa2ca86dde8222c779312904fc66bf4edb6d Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 12 Dec 2022 14:15:27 +0000 Subject: [PATCH 02/11] =?UTF-8?q?It's=20a=20fragment=20=F0=9F=99=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cluster/ClusterStateDiffIT.java | 2 +- .../common/xcontent/ChunkedToXContent.java | 24 +++++++++++++++++++ .../elasticsearch/test/ESIntegTestCase.java | 4 ++-- .../elasticsearch/test/XContentTestUtils.java | 4 ++-- .../ml/integration/MlNativeIntegTestCase.java | 4 ++-- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index a85713b8fe756..7facb922c1e61 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -162,7 +162,7 @@ public void testClusterStateDiffSerialization() throws Exception { assertThat(clusterStateFromDiffs.metadata().equalsAliases(clusterState.metadata()), is(true)); // JSON Serialization test - make sure that both states produce similar JSON - assertNull(differenceBetweenMapsIgnoringArrayOrder(convertToMap(clusterStateFromDiffs), convertToMap(clusterState))); + assertNull(differenceBetweenMapsIgnoringArrayOrder(convertToMap(clusterStateFromDiffs, true), convertToMap(clusterState, true))); // Smoke test - we cannot compare bytes to bytes because some elements might get serialized in different order // however, serialized size should remain the same diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java index 9fe5eefd45889..ab9ce6ea4559d 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java @@ -12,6 +12,7 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import java.io.IOException; import java.util.Iterator; /** @@ -42,4 +43,27 @@ static ToXContentObject wrapAsXContentObject(ChunkedToXContent chunkedToXContent return builder; }; } + + /** + * Wraps the given instance in a {@link ToXContent} that will fully serialize the instance when serialized. + * @param chunkedToXContent instance to wrap + * @param isFragment whether the wrapped instance is a fragment or a well-formed XContent object + */ + static ToXContent wrapAsXContent(ChunkedToXContent chunkedToXContent, boolean isFragment) { + return new ToXContent() { + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Iterator serialization = chunkedToXContent.toXContentChunked(params); + while (serialization.hasNext()) { + serialization.next().toXContent(builder, params); + } + return builder; + } + + @Override + public boolean isFragment() { + return isFragment; + } + }; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 7eb2187b27edd..8e6d255bea9ff 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1105,7 +1105,7 @@ protected void ensureClusterStateConsistency() throws IOException { byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState); // remove local node reference masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry); - Map masterStateMap = convertToMap(masterClusterState); + Map masterStateMap = convertToMap(masterClusterState, true); int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length; String masterId = masterClusterState.nodes().getMasterNodeId(); for (Client client : cluster().getClients()) { @@ -1113,7 +1113,7 @@ protected void ensureClusterStateConsistency() throws IOException { byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState); // remove local node reference localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null, namedWriteableRegistry); - final Map localStateMap = convertToMap(localClusterState); + final Map localStateMap = convertToMap(localClusterState, true); final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length; // Check that the non-master node has the same version of the cluster state as the master and // that the master node matches the master (otherwise there is no requirement for the cluster state to match) diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java index fcc0bd7021dd7..bb8c946254d14 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java @@ -42,8 +42,8 @@ private XContentTestUtils() { } - public static Map convertToMap(ChunkedToXContent chunkedToXContent) throws IOException { - return convertToMap(ChunkedToXContent.wrapAsXContentObject(chunkedToXContent)); + public static Map convertToMap(ChunkedToXContent chunkedToXContent, boolean isFragment) throws IOException { + return convertToMap(ChunkedToXContent.wrapAsXContent(chunkedToXContent, isFragment)); } public static Map convertToMap(ToXContent part) throws IOException { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index 905cb0e0146ec..2c53b6766a0f2 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -357,7 +357,7 @@ protected void ensureClusterStateConsistency() throws IOException { byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState); // remove local node reference masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry); - Map masterStateMap = convertToMap(masterClusterState); + Map masterStateMap = convertToMap(masterClusterState, true); int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length; String masterId = masterClusterState.nodes().getMasterNodeId(); for (Client client : cluster().getClients()) { @@ -365,7 +365,7 @@ protected void ensureClusterStateConsistency() throws IOException { byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState); // remove local node reference localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null, namedWriteableRegistry); - final Map localStateMap = convertToMap(localClusterState); + final Map localStateMap = convertToMap(localClusterState, true); final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length; // Check that the non-master node has the same version of the cluster state as the master and // that the master node matches the master (otherwise there is no requirement for the cluster state to match) From 6484863ac78c6b188664155a9d7ca9b3f1f2d12d Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 12 Dec 2022 14:30:35 +0000 Subject: [PATCH 03/11] Spotless --- .../java/org/elasticsearch/cluster/ClusterStateDiffIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index 7facb922c1e61..8c9faab8d6536 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -162,7 +162,9 @@ public void testClusterStateDiffSerialization() throws Exception { assertThat(clusterStateFromDiffs.metadata().equalsAliases(clusterState.metadata()), is(true)); // JSON Serialization test - make sure that both states produce similar JSON - assertNull(differenceBetweenMapsIgnoringArrayOrder(convertToMap(clusterStateFromDiffs, true), convertToMap(clusterState, true))); + assertNull( + differenceBetweenMapsIgnoringArrayOrder(convertToMap(clusterStateFromDiffs, true), convertToMap(clusterState, true)) + ); // Smoke test - we cannot compare bytes to bytes because some elements might get serialized in different order // however, serialized size should remain the same From b4b1b5f644d567909d2fe47696dadda37c8bcf49 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 12 Dec 2022 15:26:29 +0000 Subject: [PATCH 04/11] More fragment handling --- .../elasticsearch/search/SearchCancellationIT.java | 4 ++-- .../cluster/snapshots/get/GetSnapshotsResponse.java | 2 +- .../action/admin/indices/get/GetIndexResponse.java | 2 +- .../indices/mapping/get/GetMappingsResponse.java | 2 +- .../action/fieldcaps/FieldCapabilitiesResponse.java | 2 +- .../cluster/RepositoryCleanupInProgress.java | 2 +- .../cluster/coordination/Coordinator.java | 12 ++++++++---- .../cluster/metadata/ComponentTemplateMetadata.java | 2 +- .../metadata/ComposableIndexTemplateMetadata.java | 2 +- .../cluster/metadata/DataStreamMetadata.java | 2 +- .../cluster/metadata/RepositoriesMetadata.java | 2 +- .../main/java/org/elasticsearch/common/Strings.java | 4 ++-- .../persistent/PersistentTasksCustomMetadata.java | 2 +- .../elasticsearch/snapshots/SnapshotsService.java | 2 +- .../MergedFieldCapabilitiesResponseTests.java | 2 +- .../serialization/ClusterStateToStringTests.java | 2 +- .../org/elasticsearch/xpack/core/ml/MlMetadata.java | 2 +- .../xpack/core/slm/SnapshotLifecycleMetadata.java | 2 +- .../xpack/core/transform/TransformMetadata.java | 2 +- .../xpack/ml/integration/DeleteExpiredDataIT.java | 4 ++-- .../ml/autoscaling/MlMemoryAutoscalingDecider.java | 2 +- .../TrainedModelAssignmentClusterService.java | 2 +- .../assignment/TrainedModelAssignmentMetadata.java | 2 +- 23 files changed, 33 insertions(+), 29 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java index 525b8677a0de7..6dadfb495439d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java @@ -64,7 +64,7 @@ public void testCancellationDuringQueryPhase() throws Exception { awaitForBlock(plugins); cancelSearch(SearchAction.NAME); disableBlocks(plugins); - logger.info("Segments {}", Strings.toString(client().admin().indices().prepareSegments("test").get())); + logger.info("Segments {}", Strings.toString(client().admin().indices().prepareSegments("test").get(), false)); ensureSearchWasCancelled(searchResponse); } @@ -81,7 +81,7 @@ public void testCancellationDuringFetchPhase() throws Exception { awaitForBlock(plugins); cancelSearch(SearchAction.NAME); disableBlocks(plugins); - logger.info("Segments {}", Strings.toString(client().admin().indices().prepareSegments("test").get())); + logger.info("Segments {}", Strings.toString(client().admin().indices().prepareSegments("test").get(), false)); ensureSearchWasCancelled(searchResponse); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index f80ace56b62c0..3d24d3dab4b5c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -220,6 +220,6 @@ public int hashCode() { @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index 1e96b950c7a18..114d531c9c885 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -236,7 +236,7 @@ public Iterator toXContentChunked(ToXContent.Params ignore @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index e1db53e5e3e52..00871bbfdf6be 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -95,7 +95,7 @@ public Iterator toXContentChunked(ToXContent.Params outerParams) { @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index 7cd8f8adbdd28..5a2a15fe9880f 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -241,6 +241,6 @@ public int hashCode() { @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java index 47db3993f95fc..49d2ed329c8cf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java @@ -79,7 +79,7 @@ public Iterator toXContentChunked(ToXContent.Params ignore @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 33e3528246d12..0aa4f7004615b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -1490,17 +1490,21 @@ assert getLocalNode().equals(clusterState.getNodes().get(getLocalNode().getId()) // deserialized from the resulting JSON private boolean assertPreviousStateConsistency(ClusterStatePublicationEvent clusterStatePublicationEvent) { assert clusterStatePublicationEvent.getOldState() == coordinationState.get().getLastAcceptedState() - || XContentHelper.convertToMap(JsonXContent.jsonXContent, Strings.toString(clusterStatePublicationEvent.getOldState()), false) + || XContentHelper.convertToMap( + JsonXContent.jsonXContent, + Strings.toString(clusterStatePublicationEvent.getOldState(), true), + false + ) .equals( XContentHelper.convertToMap( JsonXContent.jsonXContent, - Strings.toString(clusterStateWithNoMasterBlock(coordinationState.get().getLastAcceptedState())), + Strings.toString(clusterStateWithNoMasterBlock(coordinationState.get().getLastAcceptedState()), true), false ) ) - : Strings.toString(clusterStatePublicationEvent.getOldState()) + : Strings.toString(clusterStatePublicationEvent.getOldState(), true) + " vs " - + Strings.toString(clusterStateWithNoMasterBlock(coordinationState.get().getLastAcceptedState())); + + Strings.toString(clusterStateWithNoMasterBlock(coordinationState.get().getLastAcceptedState()), true); return true; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java index 6e1ff6e89c7e7..027cb6f8bf805 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java @@ -123,7 +123,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } static class ComponentTemplateMetadataDiff implements NamedDiff { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java index 162d128464fcb..8417855de5c89 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java @@ -124,7 +124,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } static class ComposableIndexTemplateMetadataDiff implements NamedDiff { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java index 8099c456e89e9..4d2b8997b731b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java @@ -253,7 +253,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } static class DataStreamMetadataDiff implements NamedDiff { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java index c5548e5138b79..df71c2f999f34 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java @@ -291,6 +291,6 @@ public static XContentBuilder toXContent(RepositoryMetadata repository, XContent @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } } diff --git a/server/src/main/java/org/elasticsearch/common/Strings.java b/server/src/main/java/org/elasticsearch/common/Strings.java index 0fed68bdf732a..11c968fc664d3 100644 --- a/server/src/main/java/org/elasticsearch/common/Strings.java +++ b/server/src/main/java/org/elasticsearch/common/Strings.java @@ -755,8 +755,8 @@ public static String toString(ToXContent toXContent) { * TODO: remove this method, it makes no sense to turn potentially very large chunked xcontent instances into a string */ @Deprecated - public static String toString(ChunkedToXContent chunkedToXContent) { - return toString(ChunkedToXContent.wrapAsXContentObject(chunkedToXContent)); + public static String toString(ChunkedToXContent chunkedToXContent, boolean isFragment) { + return toString(ChunkedToXContent.wrapAsXContent(chunkedToXContent, isFragment)); } /** diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java index 00f7fa031e125..42c4e93b48284 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java @@ -187,7 +187,7 @@ public int hashCode() { @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } public long getNumberOfTasksOnNode(String nodeId, String taskName) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index e58949b8409a9..bad624611ebf9 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1050,7 +1050,7 @@ private static boolean assertNoDanglingSnapshots(ClusterState state) { : "Found shard snapshot actively executing in [" + entry + "] when it should be blocked by a running delete [" - + Strings.toString(snapshotDeletionsInProgress) + + Strings.toString(snapshotDeletionsInProgress, false) + "]"; } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index 7954543c17940..823186c40aea9 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -148,7 +148,7 @@ public void testToXContent() throws IOException { } } ] - }""".replaceAll("\\s+", ""), Strings.toString(response)); + }""".replaceAll("\\s+", ""), Strings.toString(response, false)); } private static FieldCapabilitiesResponse createSimpleResponse() { diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java index b50c227e3c615..43ffb26fbfc45 100644 --- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java @@ -58,7 +58,7 @@ public void testClusterStateSerialization() throws Exception { .routingTable(strategy.reroute(clusterState, "reroute", ActionListener.noop()).routingTable()) .build(); - String clusterStateString = Strings.toString(clusterState); + String clusterStateString = Strings.toString(clusterState, true); assertNotNull(clusterStateString); assertThat(clusterStateString, containsString("test_idx")); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java index e43bf88c28874..5904ac5051978 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java @@ -202,7 +202,7 @@ public boolean equals(Object o) { @Override public final String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java index 0398382f4d8ae..76d88a2f694bc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java @@ -148,7 +148,7 @@ public Iterator toXContentChunked(ToXContent.Params ignore @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java index e5cac570a2ac0..b930e50d7a40b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java @@ -134,7 +134,7 @@ public boolean equals(Object o) { @Override public final String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } @Override diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java index 9b58cbe3a70d0..e81f693984eed 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java @@ -135,7 +135,7 @@ public void testDeleteExpiredDataActionDeletesEmptyStateIndices() throws Excepti GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(".ml-state*").get(); assertThat( - Strings.toString(getIndexResponse), + Strings.toString(getIndexResponse, false), getIndexResponse.getIndices(), is(arrayContaining(".ml-state", ".ml-state-000001", ".ml-state-000003", ".ml-state-000005", ".ml-state-000007")) ); @@ -145,7 +145,7 @@ public void testDeleteExpiredDataActionDeletesEmptyStateIndices() throws Excepti getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(".ml-state*").get(); assertThat( - Strings.toString(getIndexResponse), + Strings.toString(getIndexResponse, false), getIndexResponse.getIndices(), // Only non-empty or current indices should survive deletion process is(arrayContaining(".ml-state-000001", ".ml-state-000005", ".ml-state-000007")) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDecider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDecider.java index 5a5c89c985f01..9fbf86863089a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDecider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDecider.java @@ -236,7 +236,7 @@ public MlMemoryAutoscalingCapacity scale(Settings configuration, AutoscalingDeci logger.debug( () -> format( "persistent tasks that caused unexpected scaling situation: [%s]", - (mlContext.persistentTasks == null) ? "null" : Strings.toString(mlContext.persistentTasks) + (mlContext.persistentTasks == null) ? "null" : Strings.toString(mlContext.persistentTasks, false) ) ); return refreshMemoryTrackerAndBuildEmptyDecision( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java index 2d4b73b24baf8..811a3c8e15f54 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java @@ -729,7 +729,7 @@ static ClusterState updateModelRoutingTable(ClusterState currentState, UpdateTra final String modelId = request.getModelId(); final String nodeId = request.getNodeId(); TrainedModelAssignmentMetadata metadata = TrainedModelAssignmentMetadata.fromState(currentState); - logger.trace(() -> format("[%s] [%s] current metadata before update %s", modelId, nodeId, Strings.toString(metadata))); + logger.trace(() -> format("[%s] [%s] current metadata before update %s", modelId, nodeId, Strings.toString(metadata, false))); final TrainedModelAssignment existingAssignment = metadata.getModelAssignment(modelId); final TrainedModelAssignmentMetadata.Builder builder = TrainedModelAssignmentMetadata.builder(currentState); // If state is stopped, this indicates the node process is closed, remove the node from the assignment diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadata.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadata.java index f3d311719fc1f..b31daab377987 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadata.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadata.java @@ -155,7 +155,7 @@ public int hashCode() { @Override public String toString() { - return Strings.toString(this); + return Strings.toString(this, false); } public boolean hasOutdatedAssignments() { From 2887dded4967397381148316ced79432dc58753c Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 12 Dec 2022 16:22:24 +0000 Subject: [PATCH 05/11] mocking a ClusterState? --- .../collector/cluster/ClusterStatsMonitoringDocTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index 0b51bc3e63a7c..204bc3af14de4 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -75,6 +75,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -119,6 +120,7 @@ public void setUp() throws Exception { .add(masterNode); when(clusterState.nodes()).thenReturn(builder.build()); + when(clusterState.toXContentChunked(any())).thenReturn(Collections.emptyIterator()); } @Override From ca54c40f8ff2184c739c6a14aee2c0c9b7263d4d Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 12 Dec 2022 16:26:25 +0000 Subject: [PATCH 06/11] Handle chunked body --- .../snapshots/DedicatedClusterSnapshotRestoreIT.java | 7 +++++-- .../java/org/elasticsearch/rest/RestResponseUtils.java | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 964b7ea976323..5707a9e8d5448 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.seqno.RetentionLeaseActions; @@ -51,6 +52,7 @@ import org.elasticsearch.rest.AbstractRestChannel; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestResponseUtils; import org.elasticsearch.rest.action.admin.cluster.RestClusterStateAction; import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction; import org.elasticsearch.snapshots.mockstore.MockRepository; @@ -525,8 +527,9 @@ public void sendResponse(RestResponse response) { @Override public void sendResponse(RestResponse response) { try { - assertThat(response.content().utf8ToString(), containsString("notsecretusername")); - assertThat(response.content().utf8ToString(), not(containsString("verysecretpassword"))); + final var responseBody = RestResponseUtils.getBodyContent(response).utf8ToString(); + assertThat(responseBody, containsString("notsecretusername")); + assertThat(responseBody, not(containsString("verysecretpassword"))); } catch (AssertionError ex) { clusterStateError.set(ex); } finally { diff --git a/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java b/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java index 47b679e894635..c13abb3b969cf 100644 --- a/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java @@ -18,7 +18,7 @@ public class RestResponseUtils { private RestResponseUtils() {} - public static BytesReference getBodyContent(RestResponse restResponse) throws IOException { + public static BytesReference getBodyContent(RestResponse restResponse) { if (restResponse.isChunked() == false) { return restResponse.content(); } @@ -40,6 +40,8 @@ public static BytesReference getBodyContent(RestResponse restResponse) throws IO out.flush(); return out.bytes(); + } catch (Exception e) { + throw new AssertionError("unexpected", e); } } } From ebcd3225add1d2345fd2a37e493fbdbda11b5323 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 12 Dec 2022 16:27:03 +0000 Subject: [PATCH 07/11] Spotless --- .../snapshots/DedicatedClusterSnapshotRestoreIT.java | 1 - .../src/main/java/org/elasticsearch/rest/RestResponseUtils.java | 2 -- 2 files changed, 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 5707a9e8d5448..85c453ad6be3c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -37,7 +37,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.seqno.RetentionLeaseActions; diff --git a/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java b/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java index c13abb3b969cf..cd0d541044195 100644 --- a/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java @@ -11,8 +11,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import java.io.IOException; - import static org.elasticsearch.transport.BytesRefRecycler.NON_RECYCLING_INSTANCE; public class RestResponseUtils { From 214e75db122a8aba3b0a5f645f16ee952047f0c2 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 29 Dec 2022 21:40:37 +0000 Subject: [PATCH 08/11] Inline single-use util --- .../main/java/org/elasticsearch/cluster/ClusterState.java | 6 +++--- .../common/xcontent/ChunkedToXContentHelper.java | 4 ---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index aa7cb0184e04a..82eead95e2e58 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -537,16 +537,16 @@ public Iterator toXContentChunked(ToXContent.Params outerP return Iterators.concat( // always provide the cluster_uuid as part of the top-level response (also part of the metadata response) - ChunkedToXContentHelper.field("cluster_uuid", metadata().clusterUUID()), + Iterators.single(((builder, params) -> builder.field("cluster_uuid", metadata().clusterUUID()))), // state version info metrics.contains(Metric.VERSION) - ? Iterators.single((builder2, params2) -> builder2.field("version", version).field("state_uuid", stateUUID)) + ? Iterators.single((builder, params) -> builder.field("version", version).field("state_uuid", stateUUID)) : Collections.emptyIterator(), // master node metrics.contains(Metric.MASTER_NODE) - ? Iterators.single((builder1, params1) -> builder1.field("master_node", nodes().getMasterNodeId())) + ? Iterators.single((builder, params) -> builder.field("master_node", nodes().getMasterNodeId())) : Collections.emptyIterator(), // blocks diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java index 4a52024f137fc..19ebbe350a53f 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java @@ -62,10 +62,6 @@ public static Iterator field(String name, boolean value) { return Iterators.single(((builder, params) -> builder.field(name, value))); } - public static Iterator field(String name, String value) { - return Iterators.single(((builder, params) -> builder.field(name, value))); - } - public static Iterator array(String name, Iterator contents) { return Iterators.concat(ChunkedToXContentHelper.startArray(name), contents, ChunkedToXContentHelper.endArray()); } From e26979e940e1aca880cca17ebb46ab4689fdebb9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 29 Dec 2022 21:50:02 +0000 Subject: [PATCH 09/11] Combine one-shot chunks --- .../elasticsearch/cluster/ClusterState.java | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 82eead95e2e58..45528fa44c961 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -536,18 +536,25 @@ public Iterator toXContentChunked(ToXContent.Params outerP final var metrics = Metric.parseString(outerParams.param("metric", "_all"), true); return Iterators.concat( - // always provide the cluster_uuid as part of the top-level response (also part of the metadata response) - Iterators.single(((builder, params) -> builder.field("cluster_uuid", metadata().clusterUUID()))), - - // state version info - metrics.contains(Metric.VERSION) - ? Iterators.single((builder, params) -> builder.field("version", version).field("state_uuid", stateUUID)) - : Collections.emptyIterator(), - - // master node - metrics.contains(Metric.MASTER_NODE) - ? Iterators.single((builder, params) -> builder.field("master_node", nodes().getMasterNodeId())) - : Collections.emptyIterator(), + + // header chunk + Iterators.single(((builder, params) -> { + // always provide the cluster_uuid as part of the top-level response (also part of the metadata response) + builder.field("cluster_uuid", metadata().clusterUUID()); + + // state version info + if (metrics.contains(Metric.VERSION)) { + builder.field("version", version); + builder.field("state_uuid", stateUUID); + } + + // master node + if (metrics.contains(Metric.MASTER_NODE)) { + builder.field("master_node", nodes().getMasterNodeId()); + } + + return builder; + })), // blocks chunkedSection(metrics.contains(Metric.BLOCKS), (builder, params) -> { From b76930f79bb16ae78994da5621d585536143013c Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 29 Dec 2022 22:26:08 +0000 Subject: [PATCH 10/11] Fix chunk counts --- .../java/org/elasticsearch/cluster/ClusterStateTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index cc461ac44b1ab..de5316ef63992 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -142,7 +142,7 @@ public void testToXContent() throws IOException { clusterState, builder, new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API)), - 36 + 34 ); builder.endObject(); @@ -374,7 +374,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); builder.startObject(); - writeChunks(clusterState, builder, new ToXContent.MapParams(mapParams), 36); + writeChunks(clusterState, builder, new ToXContent.MapParams(mapParams), 34); builder.endObject(); assertEquals(formatted(""" @@ -602,7 +602,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); builder.startObject(); - writeChunks(clusterState, builder, new ToXContent.MapParams(mapParams), 36); + writeChunks(clusterState, builder, new ToXContent.MapParams(mapParams), 34); builder.endObject(); assertEquals(formatted(""" @@ -854,7 +854,7 @@ public void testToXContentSameTypeName() throws IOException { XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); builder.startObject(); - writeChunks(clusterState, builder, ToXContent.EMPTY_PARAMS, 27); + writeChunks(clusterState, builder, ToXContent.EMPTY_PARAMS, 25); builder.endObject(); assertEquals(formatted(""" From c5bc46d577a5dcb5e01986b36d18462af07cc1c5 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 30 Dec 2022 21:32:49 +0000 Subject: [PATCH 11/11] wrapWithObject --- .../main/java/org/elasticsearch/cluster/ClusterState.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 45528fa44c961..3ed1e7e19aee4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -640,11 +640,7 @@ public Iterator toXContentChunked(ToXContent.Params outerP metrics.contains(Metric.CUSTOMS) ? Iterators.flatMap( customs.entrySet().iterator(), - cursor -> Iterators.concat( - ChunkedToXContentHelper.startObject(cursor.getKey()), - cursor.getValue().toXContentChunked(outerParams), - ChunkedToXContentHelper.endObject() - ) + cursor -> ChunkedToXContentHelper.wrapWithObject(cursor.getKey(), cursor.getValue().toXContentChunked(outerParams)) ) : Collections.emptyIterator() );