Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a cluster setting to disallow loading fielddata on _id field #49166

Merged
merged 5 commits into from
Nov 27, 2019
Merged

Add a cluster setting to disallow loading fielddata on _id field #49166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2768165
Add a cluster setting to disallow loading fielddata on _id field
jimczi Nov 15, 2019
537db09
fix security tests
jimczi Nov 18, 2019
edd4cb6
Update server/src/main/java/org/elasticsearch/index/mapper/IdFieldMap…
jimczi Nov 22, 2019
e8aa2c4
address review
jimczi Nov 22, 2019
ff3cf6e
Merge branch 'master' into disable_id_fielddata
jimczi Nov 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 19 additions & 18 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,28 +114,28 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
public void indexDocuments() throws IOException {
{
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1");
doc1.setJsonEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}");
doc1.setJsonEntity("{\"type\":\"type1\", \"id\":1, \"num\":10, \"num2\":50}");
client().performRequest(doc1);
Request doc2 = new Request(HttpPut.METHOD_NAME, "/index/_doc/2");
doc2.setJsonEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}");
doc2.setJsonEntity("{\"type\":\"type1\", \"id\":2, \"num\":20, \"num2\":40}");
client().performRequest(doc2);
Request doc3 = new Request(HttpPut.METHOD_NAME, "/index/_doc/3");
doc3.setJsonEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}");
doc3.setJsonEntity("{\"type\":\"type1\", \"id\":3, \"num\":50, \"num2\":35}");
client().performRequest(doc3);
Request doc4 = new Request(HttpPut.METHOD_NAME, "/index/_doc/4");
doc4.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}");
doc4.setJsonEntity("{\"type\":\"type2\", \"id\":4, \"num\":100, \"num2\":10}");
client().performRequest(doc4);
Request doc5 = new Request(HttpPut.METHOD_NAME, "/index/_doc/5");
doc5.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}");
doc5.setJsonEntity("{\"type\":\"type2\", \"id\":5, \"num\":100, \"num2\":10}");
client().performRequest(doc5);
}

{
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/1");
doc1.setJsonEntity("{\"field\":\"value1\", \"rating\": 7}");
doc1.setJsonEntity("{\"id\":1, \"field\":\"value1\", \"rating\": 7}");
client().performRequest(doc1);
Request doc2 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/2");
doc2.setJsonEntity("{\"field\":\"value2\"}");
doc2.setJsonEntity("{\"id\":2, \"field\":\"value2\"}");
client().performRequest(doc2);
}

Expand All @@ -153,19 +153,19 @@ public void indexDocuments() throws IOException {
"}");
client().performRequest(create);
Request doc3 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/3");
doc3.setJsonEntity("{\"field\":\"value1\", \"rating\": \"good\"}");
doc3.setJsonEntity("{\"id\":3, \"field\":\"value1\", \"rating\": \"good\"}");
client().performRequest(doc3);
Request doc4 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/4");
doc4.setJsonEntity("{\"field\":\"value2\"}");
doc4.setJsonEntity("{\"id\":4, \"field\":\"value2\"}");
client().performRequest(doc4);
}

{
Request doc5 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/5");
doc5.setJsonEntity("{\"field\":\"value1\"}");
doc5.setJsonEntity("{\"id\":5, \"field\":\"value1\"}");
client().performRequest(doc5);
Request doc6 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/6");
doc6.setJsonEntity("{\"field\":\"value2\"}");
doc6.setJsonEntity("{\"id\":6, \"field\":\"value2\"}");
client().performRequest(doc6);
}

Expand All @@ -188,7 +188,7 @@ public void indexDocuments() throws IOException {
"}");
client().performRequest(create);
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index4/_doc/1");
doc1.setJsonEntity("{\"field1\":\"value1\", \"field2\":\"value2\"}");
doc1.setJsonEntity("{\"id\":1, \"field1\":\"value1\", \"field2\":\"value2\"}");
client().performRequest(doc1);

Request createFilteredAlias = new Request(HttpPost.METHOD_NAME, "/_aliases");
Expand Down Expand Up @@ -225,7 +225,7 @@ public void testSearchNoQuery() throws IOException {
assertEquals(1.0f, searchHit.getScore(), 0);
assertEquals(-1L, searchHit.getVersion());
assertNotNull(searchHit.getSourceAsMap());
assertEquals(3, searchHit.getSourceAsMap().size());
assertEquals(4, searchHit.getSourceAsMap().size());
assertTrue(searchHit.getSourceAsMap().containsKey("type"));
assertTrue(searchHit.getSourceAsMap().containsKey("num"));
assertTrue(searchHit.getSourceAsMap().containsKey("num2"));
Expand All @@ -249,7 +249,7 @@ public void testSearchMatchQuery() throws IOException {
assertThat(searchHit.getScore(), greaterThan(0f));
assertEquals(-1L, searchHit.getVersion());
assertNotNull(searchHit.getSourceAsMap());
assertEquals(3, searchHit.getSourceAsMap().size());
assertEquals(4, searchHit.getSourceAsMap().size());
assertEquals("type1", searchHit.getSourceAsMap().get("type"));
assertEquals(50, searchHit.getSourceAsMap().get("num2"));
}
Expand Down Expand Up @@ -705,13 +705,13 @@ public void testSearchScroll() throws Exception {
public void testMultiSearch() throws Exception {
MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
SearchRequest searchRequest1 = new SearchRequest("index1");
searchRequest1.source().sort("_id", SortOrder.ASC);
searchRequest1.source().sort("id", SortOrder.ASC);
multiSearchRequest.add(searchRequest1);
SearchRequest searchRequest2 = new SearchRequest("index2");
searchRequest2.source().sort("_id", SortOrder.ASC);
searchRequest2.source().sort("id", SortOrder.ASC);
multiSearchRequest.add(searchRequest2);
SearchRequest searchRequest3 = new SearchRequest("index3");
searchRequest3.source().sort("_id", SortOrder.ASC);
searchRequest3.source().sort("id", SortOrder.ASC);
multiSearchRequest.add(searchRequest3);

MultiSearchResponse multiSearchResponse =
Expand Down Expand Up @@ -1198,7 +1198,8 @@ public void testExplainWithFetchSource() throws IOException {
assertTrue(explainResponse.hasExplanation());
assertThat(explainResponse.getExplanation().getValue(), equalTo(1.0f));
assertTrue(explainResponse.getGetResult().isExists());
assertThat(explainResponse.getGetResult().getSource(), equalTo(Collections.singletonMap("field1", "value1")));
assertEquals(2, explainResponse.getGetResult().getSource().size());
assertThat(explainResponse.getGetResult().getSource().get("field1"), equalTo("value1"));
}
}

Expand Down
18 changes: 12 additions & 6 deletions ...igh-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ public void testSearch() throws Exception {

// tag::search-source-sorting
sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); // <1>
sourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC)); // <2>
sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); // <2>
// end::search-source-sorting

// tag::search-source-filtering-off
Expand Down Expand Up @@ -1251,6 +1251,9 @@ private void indexSearchTestData() throws IOException {
CreateIndexRequest authorsRequest = new CreateIndexRequest("authors")
.mapping(XContentFactory.jsonBuilder().startObject()
.startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("user")
.field("type", "keyword")
.field("doc_values", "false")
Expand All @@ -1263,6 +1266,9 @@ private void indexSearchTestData() throws IOException {
CreateIndexRequest reviewersRequest = new CreateIndexRequest("contributors")
.mapping(XContentFactory.jsonBuilder().startObject()
.startObject("properties")
.startObject("id")
.field("type", "keyword")
.endObject()
.startObject("user")
.field("type", "keyword")
.field("store", "true")
Expand All @@ -1274,19 +1280,19 @@ private void indexSearchTestData() throws IOException {

BulkRequest bulkRequest = new BulkRequest();
bulkRequest.add(new IndexRequest("posts").id("1")
.source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user",
.source(XContentType.JSON, "id", 1, "title", "In which order are my Elasticsearch queries executed?", "user",
Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value")));
bulkRequest.add(new IndexRequest("posts").id("2")
.source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user",
.source(XContentType.JSON, "id", 2, "title", "Current status and upcoming changes in Elasticsearch", "user",
Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value")));
bulkRequest.add(new IndexRequest("posts").id("3")
.source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user",
.source(XContentType.JSON, "id", 3, "title", "The Future of Federated Search in Elasticsearch", "user",
Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value")));

bulkRequest.add(new IndexRequest("authors").id("1")
.source(XContentType.JSON, "user", "kimchy"));
.source(XContentType.JSON, "id", 1, "user", "kimchy"));
bulkRequest.add(new IndexRequest("contributors").id("1")
.source(XContentType.JSON, "user", "tanguy"));
.source(XContentType.JSON, "id", 1, "user", "tanguy"));


bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
Expand Down
15 changes: 14 additions & 1 deletion docs/reference/mapping/types/parent-join.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ PUT my_index
{
"mappings": {
"properties": {
"my_id": {
"type": "keyword"
},
"my_join_field": { <1>
"type": "join",
"relations": {
Expand All @@ -38,6 +41,7 @@ For instance the following example creates two `parent` documents in the `questi
--------------------------------------------------
PUT my_index/_doc/1?refresh
{
"my_id": "1",
"text": "This is a question",
"my_join_field": {
"name": "question" <1>
Expand All @@ -46,6 +50,7 @@ PUT my_index/_doc/1?refresh

PUT my_index/_doc/2?refresh
{
"my_id": "2",
"text": "This is another question",
"my_join_field": {
"name": "question"
Expand All @@ -63,12 +68,14 @@ as a shortcut instead of encapsulating it in the normal object notation:
--------------------------------------------------
PUT my_index/_doc/1?refresh
{
"my_id": "1",
"text": "This is a question",
"my_join_field": "question" <1>
}

PUT my_index/_doc/2?refresh
{
"my_id": "2",
"text": "This is another question",
"my_join_field": "question"
}
Expand All @@ -89,6 +96,7 @@ For instance the following example shows how to index two `child` documents:
--------------------------------------------------
PUT my_index/_doc/3?routing=1&refresh <1>
{
"my_id": "3",
"text": "This is an answer",
"my_join_field": {
"name": "answer", <2>
Expand All @@ -98,6 +106,7 @@ PUT my_index/_doc/3?routing=1&refresh <1>

PUT my_index/_doc/4?routing=1&refresh
{
"my_id": "4",
"text": "This is another answer",
"my_join_field": {
"name": "answer",
Expand Down Expand Up @@ -159,7 +168,7 @@ GET my_index/_search
"query": {
"match_all": {}
},
"sort": ["_id"]
"sort": ["my_id"]
}
--------------------------
// TEST[continued]
Expand All @@ -182,6 +191,7 @@ Will return:
"_id": "1",
"_score": null,
"_source": {
"my_id": "1",
"text": "This is a question",
"my_join_field": "question" <1>
},
Expand All @@ -194,6 +204,7 @@ Will return:
"_id": "2",
"_score": null,
"_source": {
"my_id": "2",
"text": "This is another question",
"my_join_field": "question" <2>
},
Expand All @@ -207,6 +218,7 @@ Will return:
"_score": null,
"_routing": "1",
"_source": {
"my_id": "3",
"text": "This is an answer",
"my_join_field": {
"name": "answer", <3>
Expand All @@ -223,6 +235,7 @@ Will return:
"_score": null,
"_routing": "1",
"_source": {
"my_id": "4",
"text": "This is another answer",
"my_join_field": {
"name": "answer",
Expand Down
38 changes: 22 additions & 16 deletions ...ang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,7 @@ private SearchRequestBuilder buildRequest(String script, Object... params) {

SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addSort(SortBuilders.fieldSort("_id")
.order(SortOrder.ASC))
.addSort(SortBuilders.fieldSort("id").order(SortOrder.ASC).unmappedType("long"))
.addScriptField("foo", new Script(ScriptType.INLINE, "expression", script, paramsMap));
return req;
}
Expand Down Expand Up @@ -147,8 +146,10 @@ public void testDateMethods() throws Exception {
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
ensureGreen("test");
indexRandom(true,
client().prepareIndex("test").setId("1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
client().prepareIndex("test").setId("2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
client().prepareIndex("test").setId("1")
.setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
client().prepareIndex("test").setId("2")
.setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
SearchResponse rsp = buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()").get();
assertEquals(2, rsp.getHits().getTotalHits().value);
SearchHits hits = rsp.getHits();
Expand All @@ -175,8 +176,10 @@ public void testDateObjectMethods() throws Exception {
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
ensureGreen("test");
indexRandom(true,
client().prepareIndex("test").setId("1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
client().prepareIndex("test").setId("2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
client().prepareIndex("test").setId("1")
.setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
client().prepareIndex("test").setId("2")
.setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
SearchResponse rsp = buildRequest("doc['date0'].date.secondOfMinute - doc['date0'].date.minuteOfHour").get();
assertEquals(2, rsp.getHits().getTotalHits().value);
SearchHits hits = rsp.getHits();
Expand Down Expand Up @@ -207,15 +210,18 @@ public void testMultiValueMethods() throws Exception {
ensureGreen("test");

Map<String, Object> doc1 = new HashMap<>();
doc1.put("id", 1);
doc1.put("double0", new Double[]{5.0d, 1.0d, 1.5d});
doc1.put("double1", new Double[]{1.2d, 2.4d});
doc1.put("double2", 3.0d);

Map<String, Object> doc2 = new HashMap<>();
doc2.put("id", 2);
doc2.put("double0", 5.0d);
doc2.put("double1", 3.0d);

Map<String, Object> doc3 = new HashMap<>();
doc3.put("id", 3);
doc3.put("double0", new Double[]{5.0d, 1.0d, 1.5d, -1.5d});
doc3.put("double1", 4.0d);

Expand Down Expand Up @@ -319,8 +325,8 @@ public void testSparseField() throws Exception {
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long"));
ensureGreen("test");
indexRandom(true,
client().prepareIndex("test").setId("1").setSource("x", 4),
client().prepareIndex("test").setId("2").setSource("y", 2));
client().prepareIndex("test").setId("1").setSource("id", 1, "x", 4),
client().prepareIndex("test").setId("2").setSource("id", 2, "y", 2));
SearchResponse rsp = buildRequest("doc['x'] + 1").get();
ElasticsearchAssertions.assertSearchResponse(rsp);
SearchHits hits = rsp.getHits();
Expand Down Expand Up @@ -348,9 +354,9 @@ public void testParams() throws Exception {
createIndex("test");
ensureGreen("test");
indexRandom(true,
client().prepareIndex("test").setId("1").setSource("x", 10),
client().prepareIndex("test").setId("2").setSource("x", 3),
client().prepareIndex("test").setId("3").setSource("x", 5));
client().prepareIndex("test").setId("1").setSource("id", 1, "x", 10),
client().prepareIndex("test").setId("2").setSource("id", 2, "x", 3),
client().prepareIndex("test").setId("3").setSource("id", 3, "x", 5));
// a = int, b = double, c = long
String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)";
SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get();
Expand Down Expand Up @@ -621,9 +627,9 @@ public void testBoolean() throws Exception {
assertAcked(prepareCreate("test").addMapping("doc", xContentBuilder));
ensureGreen();
indexRandom(true,
client().prepareIndex("test").setId("1").setSource("price", 1.0, "vip", true),
client().prepareIndex("test").setId("2").setSource("price", 2.0, "vip", false),
client().prepareIndex("test").setId("3").setSource("price", 2.0, "vip", false));
client().prepareIndex("test").setId("1").setSource("id", 1, "price", 1.0, "vip", true),
client().prepareIndex("test").setId("2").setSource("id", 2, "price", 2.0, "vip", false),
client().prepareIndex("test").setId("3").setSource("id", 3, "price", 2.0, "vip", false));
// access .value
SearchResponse rsp = buildRequest("doc['vip'].value").get();
assertSearchResponse(rsp);
Expand Down Expand Up @@ -652,8 +658,8 @@ public void testFilterScript() throws Exception {
createIndex("test");
ensureGreen("test");
indexRandom(true,
client().prepareIndex("test").setId("1").setSource("foo", 1.0),
client().prepareIndex("test").setId("2").setSource("foo", 0.0));
client().prepareIndex("test").setId("1").setSource("id", 1, "foo", 1.0),
client().prepareIndex("test").setId("2").setSource("id", 2, "foo", 0.0));
SearchRequestBuilder builder = buildRequest("doc['foo'].value");
Script script = new Script(ScriptType.INLINE, "expression", "doc['foo'].value", Collections.emptyMap());
builder.setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.scriptQuery(script)));
Expand Down
2 changes: 1 addition & 1 deletion modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ public void testParentWithMultipleBuckets() throws Exception {
.setQuery(matchQuery("randomized", false))
.addAggregation(
terms("category").field("category").size(10000).subAggregation(
children("to_comment", "comment").subAggregation(topHits("top_comments").sort("_id", SortOrder.ASC))
children("to_comment", "comment").subAggregation(topHits("top_comments").sort("id", SortOrder.ASC))
)
).get();
assertSearchResponse(searchResponse);
Expand Down
Loading