From 22b1daf392bc6b116a13998c0579ce63c4fb0ed9 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 15 Mar 2022 10:06:16 -0400 Subject: [PATCH 1/3] Add BWC test for field-caps (#84455) Relates #83494 --- .../elasticsearch/upgrades/FieldCapsIT.java | 255 ++++++++++++++++++ .../test/rest/ESRestTestCase.java | 20 ++ 2 files changed, 275 insertions(+) create mode 100644 qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java new file mode 100644 index 0000000000000..ddf61aa4e25db --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.upgrades; + +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.Before; + +import java.util.List; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; + +/** + * Since ES 8.2, field-caps internal responses are shared between indices that have the same index mapping hash to + * reduce the transport message size between nodes and clusters, and the memory usage to hold these internal responses. + * As the optimization is applied for only field-caps requests without index-filter and nodes on 8.2 or later, + * these BWC tests verify these combinations of field-caps requests: (old|new|mixed indices) and (with|without index filter) + */ +public class FieldCapsIT extends AbstractRollingTestCase { + private static boolean indicesCreated = false; + + @Before + public void setupIndices() throws Exception { + if (indicesCreated) { + return; + } + indicesCreated = true; + final String redMapping = """ + "properties": { + "red_field": { "type": "keyword" }, + "yellow_field": { "type": "integer" }, + "blue_field": { "type": "keyword" }, + "timestamp": {"type": "date"} + } + """; + final String greenMapping = """ + "properties": { + "green_field": { "type": "keyword" }, + "yellow_field": { "type": "long" }, + "blue_field": { "type": "keyword" }, + "timestamp": {"type": "date"} + } + """; + if (CLUSTER_TYPE == ClusterType.OLD) { + createIndex("old_red_1", Settings.EMPTY, redMapping); + createIndex("old_red_2", Settings.EMPTY, redMapping); + createIndex("old_red_empty", Settings.EMPTY, redMapping); + createIndex("old_green_1", Settings.EMPTY, greenMapping); + createIndex("old_green_2", Settings.EMPTY, greenMapping); + createIndex("old_green_empty", Settings.EMPTY, greenMapping); + for (String index : List.of("old_red_1", "old_red_2", "old_green_1", "old_green_2")) { + final Request indexRequest = new Request("POST", "/" + index + "/" + "_doc/1"); + indexRequest.addParameter("refresh", "true"); + indexRequest.setJsonEntity( + Strings.toString(JsonXContent.contentBuilder().startObject().field("timestamp", "2020-01-01").endObject()) + ); + assertOK(client().performRequest(indexRequest)); + } + } else if (CLUSTER_TYPE == ClusterType.MIXED && FIRST_MIXED_ROUND) { + createIndex("new_red_1", Settings.EMPTY, redMapping); + createIndex("new_red_2", Settings.EMPTY, redMapping); + createIndex("new_red_empty", Settings.EMPTY, redMapping); + createIndex("new_green_1", Settings.EMPTY, greenMapping); + createIndex("new_green_2", Settings.EMPTY, greenMapping); + createIndex("new_green_empty", Settings.EMPTY, greenMapping); + for (String index : List.of("new_red_1", "new_red_2", "new_green_1", "new_green_2")) { + final Request indexRequest = new Request("POST", "/" + index + "/" + "_doc/1"); + indexRequest.addParameter("refresh", "true"); + indexRequest.setJsonEntity( + Strings.toString(JsonXContent.contentBuilder().startObject().field("timestamp", "2020-10-10").endObject()) + ); + assertOK(client().performRequest(indexRequest)); + } + } + } + + public void testOldIndicesOnly() throws Exception { + { + FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), null); + assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2", "old_red_empty" })); + assertThat(resp.getField("red_field").keySet(), contains("keyword")); + assertTrue(resp.getField("red_field").get("keyword").isSearchable()); + assertThat(resp.getField("yellow_field").keySet(), contains("integer")); + assertTrue(resp.getField("yellow_field").get("integer").isSearchable()); + assertThat(resp.getField("blue_field").keySet(), contains("keyword")); + assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); + } + { + FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), null); + assertThat( + resp.getIndices(), + equalTo(new String[] { "old_green_1", "old_green_2", "old_green_empty", "old_red_1", "old_red_2", "old_red_empty" }) + ); + assertThat(resp.getField("red_field").keySet(), contains("keyword")); + assertTrue(resp.getField("red_field").get("keyword").isSearchable()); + assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long")); + assertTrue(resp.getField("yellow_field").get("integer").isSearchable()); + assertTrue(resp.getField("yellow_field").get("long").isSearchable()); + assertThat(resp.getField("blue_field").keySet(), contains("keyword")); + assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); + } + } + + public void testOldIndicesWithIndexFilter() throws Exception { + final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); + { + FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), indexFilter); + assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2" })); + assertThat(resp.getField("red_field").keySet(), contains("keyword")); + assertTrue(resp.getField("red_field").get("keyword").isSearchable()); + assertThat(resp.getField("yellow_field").keySet(), contains("integer")); + assertTrue(resp.getField("yellow_field").get("integer").isSearchable()); + assertThat(resp.getField("blue_field").keySet(), contains("keyword")); + assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); + } + { + FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), indexFilter); + assertThat(resp.getIndices(), equalTo(new String[] { "old_green_1", "old_green_2", "old_red_1", "old_red_2" })); + assertThat(resp.getField("red_field").keySet(), contains("keyword")); + assertTrue(resp.getField("red_field").get("keyword").isSearchable()); + assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long")); + assertTrue(resp.getField("yellow_field").get("integer").isSearchable()); + assertTrue(resp.getField("yellow_field").get("long").isSearchable()); + assertThat(resp.getField("blue_field").keySet(), contains("keyword")); + assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); + } + } + + public void testNewIndicesOnly() throws Exception { + assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); + { + FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), null); + assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2", "new_red_empty" })); + assertThat(resp.getField("red_field").keySet(), contains("keyword")); + assertTrue(resp.getField("red_field").get("keyword").isSearchable()); + assertThat(resp.getField("yellow_field").keySet(), contains("integer")); + assertTrue(resp.getField("yellow_field").get("integer").isSearchable()); + assertThat(resp.getField("blue_field").keySet(), contains("keyword")); + assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); + } + { + FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), null); + assertThat( + resp.getIndices(), + equalTo(new String[] { "new_green_1", "new_green_2", "new_green_empty", "new_red_1", "new_red_2", "new_red_empty" }) + ); + assertThat(resp.getField("red_field").keySet(), contains("keyword")); + assertTrue(resp.getField("red_field").get("keyword").isSearchable()); + assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long")); + assertTrue(resp.getField("yellow_field").get("integer").isSearchable()); + assertTrue(resp.getField("yellow_field").get("long").isSearchable()); + assertThat(resp.getField("blue_field").keySet(), contains("keyword")); + assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); + } + } + + public void testNewIndicesOnlyWithIndexFilter() throws Exception { + assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); + final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); + { + FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), indexFilter); + assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2" })); + assertThat(resp.getField("red_field").keySet(), contains("keyword")); + assertTrue(resp.getField("red_field").get("keyword").isSearchable()); + assertThat(resp.getField("yellow_field").keySet(), contains("integer")); + assertTrue(resp.getField("yellow_field").get("integer").isSearchable()); + assertThat(resp.getField("blue_field").keySet(), contains("keyword")); + assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); + } + { + FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), indexFilter); + assertThat(resp.getIndices(), equalTo(new String[] { "new_green_1", "new_green_2", "new_red_1", "new_red_2" })); + assertThat(resp.getField("red_field").keySet(), contains("keyword")); + assertTrue(resp.getField("red_field").get("keyword").isSearchable()); + assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long")); + assertTrue(resp.getField("yellow_field").get("integer").isSearchable()); + assertTrue(resp.getField("yellow_field").get("long").isSearchable()); + assertThat(resp.getField("blue_field").keySet(), contains("keyword")); + assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); + } + } + + public void testAllIndices() throws Exception { + assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); + FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), null); + assertThat( + resp.getIndices(), + equalTo( + new String[] { + "new_green_1", + "new_green_2", + "new_green_empty", + "new_red_1", + "new_red_2", + "new_red_empty", + "old_green_1", + "old_green_2", + "old_green_empty", + "old_red_1", + "old_red_2", + "old_red_empty" } + ) + ); + assertThat(resp.getField("red_field").keySet(), contains("keyword")); + assertTrue(resp.getField("red_field").get("keyword").isSearchable()); + assertThat(resp.getField("green_field").keySet(), contains("keyword")); + assertTrue(resp.getField("green_field").get("keyword").isSearchable()); + assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long")); + assertTrue(resp.getField("yellow_field").get("integer").isSearchable()); + assertTrue(resp.getField("yellow_field").get("long").isSearchable()); + assertThat(resp.getField("blue_field").keySet(), contains("keyword")); + assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); + } + + public void testAllIndicesWithIndexFilter() throws Exception { + assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); + final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); + FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), indexFilter); + assertThat( + resp.getIndices(), + equalTo( + new String[] { + "new_green_1", + "new_green_2", + "new_red_1", + "new_red_2", + "old_green_1", + "old_green_2", + "old_red_1", + "old_red_2" } + ) + ); + assertThat(resp.getField("red_field").keySet(), contains("keyword")); + assertTrue(resp.getField("red_field").get("keyword").isSearchable()); + assertThat(resp.getField("green_field").keySet(), contains("keyword")); + assertTrue(resp.getField("green_field").get("keyword").isSearchable()); + assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long")); + assertTrue(resp.getField("yellow_field").get("integer").isSearchable()); + assertTrue(resp.getField("yellow_field").get("long").isSearchable()); + assertThat(resp.getField("blue_field").keySet(), contains("keyword")); + assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index d2f2927118a85..17883c15b263d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -25,6 +25,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RequestOptions.Builder; @@ -48,6 +49,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.indices.flush.SyncedFlushService; import org.elasticsearch.rest.RestStatus; @@ -1976,4 +1978,22 @@ protected static boolean isNotFoundResponseException(IOException ioe) { } return false; } + + protected FieldCapabilitiesResponse fieldCaps(List indices, List fields, QueryBuilder indexFilter) throws IOException { + Request request = new Request("POST", "/_field_caps"); + request.addParameter("index", String.join(",", indices)); + request.addParameter("fields", String.join(",", fields)); + if (indexFilter != null) { + XContentBuilder body = JsonXContent.contentBuilder(); + body.startObject(); + body.field("index_filter", indexFilter); + body.endObject(); + request.setJsonEntity(Strings.toString(body)); + } + Response response = client().performRequest(request); + assertOK(response); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, response.getEntity().getContent())) { + return FieldCapabilitiesResponse.fromXContent(parser); + } + } } From 25ff3e1459ad9b6ae0692f2d85d9338edf3fb674 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 15 Mar 2022 13:19:30 -0400 Subject: [PATCH 2/3] fix tests --- .../elasticsearch/upgrades/FieldCapsIT.java | 65 +++++++++---------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java index ddf61aa4e25db..2f176cacd232b 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java @@ -17,16 +17,15 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.Before; -import java.util.List; +import java.util.Arrays; +import java.util.Collections; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; /** - * Since ES 8.2, field-caps internal responses are shared between indices that have the same index mapping hash to - * reduce the transport message size between nodes and clusters, and the memory usage to hold these internal responses. - * As the optimization is applied for only field-caps requests without index-filter and nodes on 8.2 or later, - * these BWC tests verify these combinations of field-caps requests: (old|new|mixed indices) and (with|without index filter) + * Since ES 7.16, shard-level field-caps requests are batched in node-level requests. + * These BWC tests verify these combinations of field-caps requests: (old|new|mixed indices) and (with|without index filter) */ public class FieldCapsIT extends AbstractRollingTestCase { private static boolean indicesCreated = false; @@ -37,22 +36,22 @@ public void setupIndices() throws Exception { return; } indicesCreated = true; - final String redMapping = """ - "properties": { - "red_field": { "type": "keyword" }, - "yellow_field": { "type": "integer" }, - "blue_field": { "type": "keyword" }, - "timestamp": {"type": "date"} - } - """; - final String greenMapping = """ - "properties": { - "green_field": { "type": "keyword" }, - "yellow_field": { "type": "long" }, - "blue_field": { "type": "keyword" }, - "timestamp": {"type": "date"} - } - """; + final String redMapping = "" + + "\"properties\": {" + + "\"red_field\": { \"type\": \"keyword\" }," + + "\"yellow_field\": { \"type\": \"integer\" }," + + "\"blue_field\": { \"type\": \"keyword\" }," + + "\"timestamp\": {\"type\": \"date\"}" + + "}"; + + final String greenMapping = "" + + "\"properties\": {" + + "\"green_field\": { \"type\": \"keyword\" }," + + "\"yellow_field\": { \"type\": \"long\" }," + + "\"blue_field\": { \"type\": \"keyword\" }," + + "\"timestamp\": {\"type\": \"date\"}" + + "}"; + if (CLUSTER_TYPE == ClusterType.OLD) { createIndex("old_red_1", Settings.EMPTY, redMapping); createIndex("old_red_2", Settings.EMPTY, redMapping); @@ -60,7 +59,7 @@ public void setupIndices() throws Exception { createIndex("old_green_1", Settings.EMPTY, greenMapping); createIndex("old_green_2", Settings.EMPTY, greenMapping); createIndex("old_green_empty", Settings.EMPTY, greenMapping); - for (String index : List.of("old_red_1", "old_red_2", "old_green_1", "old_green_2")) { + for (String index : Arrays.asList("old_red_1", "old_red_2", "old_green_1", "old_green_2")) { final Request indexRequest = new Request("POST", "/" + index + "/" + "_doc/1"); indexRequest.addParameter("refresh", "true"); indexRequest.setJsonEntity( @@ -75,7 +74,7 @@ public void setupIndices() throws Exception { createIndex("new_green_1", Settings.EMPTY, greenMapping); createIndex("new_green_2", Settings.EMPTY, greenMapping); createIndex("new_green_empty", Settings.EMPTY, greenMapping); - for (String index : List.of("new_red_1", "new_red_2", "new_green_1", "new_green_2")) { + for (String index : Arrays.asList("new_red_1", "new_red_2", "new_green_1", "new_green_2")) { final Request indexRequest = new Request("POST", "/" + index + "/" + "_doc/1"); indexRequest.addParameter("refresh", "true"); indexRequest.setJsonEntity( @@ -88,7 +87,7 @@ public void setupIndices() throws Exception { public void testOldIndicesOnly() throws Exception { { - FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), null); + FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("old_red_*"), Collections.singletonList("*"), null); assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2", "old_red_empty" })); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); @@ -98,7 +97,7 @@ public void testOldIndicesOnly() throws Exception { assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); } { - FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), null); + FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("old_*"), Collections.singletonList("*"), null); assertThat( resp.getIndices(), equalTo(new String[] { "old_green_1", "old_green_2", "old_green_empty", "old_red_1", "old_red_2", "old_red_empty" }) @@ -116,7 +115,7 @@ public void testOldIndicesOnly() throws Exception { public void testOldIndicesWithIndexFilter() throws Exception { final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); { - FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), indexFilter); + FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("old_red_*"), Collections.singletonList("*"), indexFilter); assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2" })); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); @@ -126,7 +125,7 @@ public void testOldIndicesWithIndexFilter() throws Exception { assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); } { - FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), indexFilter); + FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("old_*"), Collections.singletonList("*"), indexFilter); assertThat(resp.getIndices(), equalTo(new String[] { "old_green_1", "old_green_2", "old_red_1", "old_red_2" })); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); @@ -141,7 +140,7 @@ public void testOldIndicesWithIndexFilter() throws Exception { public void testNewIndicesOnly() throws Exception { assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); { - FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), null); + FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("new_red_*"), Collections.singletonList("*"), null); assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2", "new_red_empty" })); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); @@ -151,7 +150,7 @@ public void testNewIndicesOnly() throws Exception { assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); } { - FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), null); + FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("new_*"), Collections.singletonList("*"), null); assertThat( resp.getIndices(), equalTo(new String[] { "new_green_1", "new_green_2", "new_green_empty", "new_red_1", "new_red_2", "new_red_empty" }) @@ -170,7 +169,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception { assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); { - FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), indexFilter); + FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("new_red_*"), Collections.singletonList("*"), indexFilter); assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2" })); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); @@ -180,7 +179,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception { assertTrue(resp.getField("blue_field").get("keyword").isSearchable()); } { - FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), indexFilter); + FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("new_*"), Collections.singletonList("*"), indexFilter); assertThat(resp.getIndices(), equalTo(new String[] { "new_green_1", "new_green_2", "new_red_1", "new_red_2" })); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); @@ -194,7 +193,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception { public void testAllIndices() throws Exception { assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); - FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), null); + FieldCapabilitiesResponse resp = fieldCaps(Arrays.asList("old_*", "new_*"), Collections.singletonList("*"), null); assertThat( resp.getIndices(), equalTo( @@ -227,7 +226,7 @@ public void testAllIndices() throws Exception { public void testAllIndicesWithIndexFilter() throws Exception { assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); - FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), indexFilter); + FieldCapabilitiesResponse resp = fieldCaps(Arrays.asList("old_*", "new_*"), Collections.singletonList("*"), indexFilter); assertThat( resp.getIndices(), equalTo( From 1fb07312ba7ec69bbc31966d9556f7ddecda475f Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 15 Mar 2022 16:42:55 -0400 Subject: [PATCH 3/3] fix tests --- .../elasticsearch/upgrades/FieldCapsIT.java | 137 ++++++++++-------- 1 file changed, 79 insertions(+), 58 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java index 2f176cacd232b..e36fd98b82e69 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java @@ -8,8 +8,11 @@ package org.elasticsearch.upgrades; +import org.elasticsearch.Version; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.client.Request; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilder; @@ -17,6 +20,7 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.Before; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; @@ -36,29 +40,49 @@ public void setupIndices() throws Exception { return; } indicesCreated = true; - final String redMapping = "" + final String redMapping = "\"_doc\": {" + "\"properties\": {" + "\"red_field\": { \"type\": \"keyword\" }," + "\"yellow_field\": { \"type\": \"integer\" }," + "\"blue_field\": { \"type\": \"keyword\" }," + "\"timestamp\": {\"type\": \"date\"}" - + "}"; + + "}}"; - final String greenMapping = "" + final String greenMapping = "\"_doc\": {" + "\"properties\": {" + "\"green_field\": { \"type\": \"keyword\" }," + "\"yellow_field\": { \"type\": \"long\" }," + "\"blue_field\": { \"type\": \"keyword\" }," + "\"timestamp\": {\"type\": \"date\"}" - + "}"; + + "}}"; + CheckedBiConsumer createIndex = (index, mapping) -> { + Request request = new Request("PUT", "/" + index); + request.addParameter("include_type_name", "true"); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5)).build(); + String entity = "{\"settings\": " + Strings.toString(settings); + if (mapping != null) { + entity += ",\"mappings\" : {" + mapping + "}"; + } + entity += "}"; + request.setJsonEntity(entity); + request.setOptions(expectVersionSpecificWarnings(v -> { + final String typesWarning = "[types removal] The parameter include_type_name should be explicitly specified in" + + " create index requests to prepare for 7.0. In 7.0 include_type_name will default to 'false', and requests" + + " are expected to omit the type name in mapping definitions."; + final String includeTypeWarning = "[types removal] Using include_type_name in create index requests is deprecated." + + " The parameter will be removed in the next major version."; + v.compatible(typesWarning, includeTypeWarning); + })); + assertOK(client().performRequest(request)); + }; if (CLUSTER_TYPE == ClusterType.OLD) { - createIndex("old_red_1", Settings.EMPTY, redMapping); - createIndex("old_red_2", Settings.EMPTY, redMapping); - createIndex("old_red_empty", Settings.EMPTY, redMapping); - createIndex("old_green_1", Settings.EMPTY, greenMapping); - createIndex("old_green_2", Settings.EMPTY, greenMapping); - createIndex("old_green_empty", Settings.EMPTY, greenMapping); + createIndex.accept("old_red_1", redMapping); + createIndex.accept("old_red_2", redMapping); + createIndex.accept("old_red_empty", redMapping); + createIndex.accept("old_green_1", greenMapping); + createIndex.accept("old_green_2", greenMapping); + createIndex.accept("old_green_empty", greenMapping); for (String index : Arrays.asList("old_red_1", "old_red_2", "old_green_1", "old_green_2")) { final Request indexRequest = new Request("POST", "/" + index + "/" + "_doc/1"); indexRequest.addParameter("refresh", "true"); @@ -68,12 +92,12 @@ public void setupIndices() throws Exception { assertOK(client().performRequest(indexRequest)); } } else if (CLUSTER_TYPE == ClusterType.MIXED && FIRST_MIXED_ROUND) { - createIndex("new_red_1", Settings.EMPTY, redMapping); - createIndex("new_red_2", Settings.EMPTY, redMapping); - createIndex("new_red_empty", Settings.EMPTY, redMapping); - createIndex("new_green_1", Settings.EMPTY, greenMapping); - createIndex("new_green_2", Settings.EMPTY, greenMapping); - createIndex("new_green_empty", Settings.EMPTY, greenMapping); + createIndex.accept("new_red_1", redMapping); + createIndex.accept("new_red_2", redMapping); + createIndex.accept("new_red_empty", redMapping); + createIndex.accept("new_green_1", greenMapping); + createIndex.accept("new_green_2", greenMapping); + createIndex.accept("new_green_empty", greenMapping); for (String index : Arrays.asList("new_red_1", "new_red_2", "new_green_1", "new_green_2")) { final Request indexRequest = new Request("POST", "/" + index + "/" + "_doc/1"); indexRequest.addParameter("refresh", "true"); @@ -85,10 +109,16 @@ public void setupIndices() throws Exception { } } + private void assertIndices(FieldCapabilitiesResponse resp, String[] expectedIndices) { + if (UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_9_0) || CLUSTER_TYPE == ClusterType.UPGRADED) { + assertThat(resp.getIndices(), equalTo(expectedIndices)); + } + } + public void testOldIndicesOnly() throws Exception { { FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("old_red_*"), Collections.singletonList("*"), null); - assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2", "old_red_empty" })); + assertIndices(resp, new String[] { "old_red_1", "old_red_2", "old_red_empty" }); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); assertThat(resp.getField("yellow_field").keySet(), contains("integer")); @@ -98,9 +128,9 @@ public void testOldIndicesOnly() throws Exception { } { FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("old_*"), Collections.singletonList("*"), null); - assertThat( - resp.getIndices(), - equalTo(new String[] { "old_green_1", "old_green_2", "old_green_empty", "old_red_1", "old_red_2", "old_red_empty" }) + assertIndices( + resp, + new String[] { "old_green_1", "old_green_2", "old_green_empty", "old_red_1", "old_red_2", "old_red_empty" } ); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); @@ -113,10 +143,11 @@ public void testOldIndicesOnly() throws Exception { } public void testOldIndicesWithIndexFilter() throws Exception { + assumeTrue("Index filter was introduced 7.9.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_9_0)); final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); { FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("old_red_*"), Collections.singletonList("*"), indexFilter); - assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2" })); + assertIndices(resp, new String[] { "old_red_1", "old_red_2" }); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); assertThat(resp.getField("yellow_field").keySet(), contains("integer")); @@ -126,7 +157,7 @@ public void testOldIndicesWithIndexFilter() throws Exception { } { FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("old_*"), Collections.singletonList("*"), indexFilter); - assertThat(resp.getIndices(), equalTo(new String[] { "old_green_1", "old_green_2", "old_red_1", "old_red_2" })); + assertIndices(resp, new String[] { "old_green_1", "old_green_2", "old_red_1", "old_red_2" }); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long")); @@ -141,7 +172,7 @@ public void testNewIndicesOnly() throws Exception { assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); { FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("new_red_*"), Collections.singletonList("*"), null); - assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2", "new_red_empty" })); + assertIndices(resp, new String[] { "new_red_1", "new_red_2", "new_red_empty" }); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); assertThat(resp.getField("yellow_field").keySet(), contains("integer")); @@ -151,9 +182,9 @@ public void testNewIndicesOnly() throws Exception { } { FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("new_*"), Collections.singletonList("*"), null); - assertThat( - resp.getIndices(), - equalTo(new String[] { "new_green_1", "new_green_2", "new_green_empty", "new_red_1", "new_red_2", "new_red_empty" }) + assertIndices( + resp, + new String[] { "new_green_1", "new_green_2", "new_green_empty", "new_red_1", "new_red_2", "new_red_empty" } ); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); @@ -166,11 +197,12 @@ public void testNewIndicesOnly() throws Exception { } public void testNewIndicesOnlyWithIndexFilter() throws Exception { + assumeTrue("Index filter was introduced 7.9.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_9_0)); assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); { FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("new_red_*"), Collections.singletonList("*"), indexFilter); - assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2" })); + assertIndices(resp, new String[] { "new_red_1", "new_red_2" }); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); assertThat(resp.getField("yellow_field").keySet(), contains("integer")); @@ -180,7 +212,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception { } { FieldCapabilitiesResponse resp = fieldCaps(Collections.singletonList("new_*"), Collections.singletonList("*"), indexFilter); - assertThat(resp.getIndices(), equalTo(new String[] { "new_green_1", "new_green_2", "new_red_1", "new_red_2" })); + assertIndices(resp, new String[] { "new_green_1", "new_green_2", "new_red_1", "new_red_2" }); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long")); @@ -194,23 +226,21 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception { public void testAllIndices() throws Exception { assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); FieldCapabilitiesResponse resp = fieldCaps(Arrays.asList("old_*", "new_*"), Collections.singletonList("*"), null); - assertThat( - resp.getIndices(), - equalTo( - new String[] { - "new_green_1", - "new_green_2", - "new_green_empty", - "new_red_1", - "new_red_2", - "new_red_empty", - "old_green_1", - "old_green_2", - "old_green_empty", - "old_red_1", - "old_red_2", - "old_red_empty" } - ) + assertIndices( + resp, + new String[] { + "new_green_1", + "new_green_2", + "new_green_empty", + "new_red_1", + "new_red_2", + "new_red_empty", + "old_green_1", + "old_green_2", + "old_green_empty", + "old_red_1", + "old_red_2", + "old_red_empty" } ); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable()); @@ -224,22 +254,13 @@ public void testAllIndices() throws Exception { } public void testAllIndicesWithIndexFilter() throws Exception { + assumeTrue("Index filter was introduced 7.9.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_9_0)); assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); FieldCapabilitiesResponse resp = fieldCaps(Arrays.asList("old_*", "new_*"), Collections.singletonList("*"), indexFilter); - assertThat( - resp.getIndices(), - equalTo( - new String[] { - "new_green_1", - "new_green_2", - "new_red_1", - "new_red_2", - "old_green_1", - "old_green_2", - "old_red_1", - "old_red_2" } - ) + assertIndices( + resp, + new String[] { "new_green_1", "new_green_2", "new_red_1", "new_red_2", "old_green_1", "old_green_2", "old_red_1", "old_red_2" } ); assertThat(resp.getField("red_field").keySet(), contains("keyword")); assertTrue(resp.getField("red_field").get("keyword").isSearchable());