From 8bc1f21dd7826bd6bb553038ff8a2bb8cb69c953 Mon Sep 17 00:00:00 2001 From: Sylvain Wallez Date: Mon, 24 Jan 2022 16:41:03 +0100 Subject: [PATCH] Add support for typed-key arrays, refactor and add tests (#125) --- .../clients/json/ExternallyTaggedUnion.java | 62 +++++++++- .../ElasticsearchTestServer.java | 111 ++++++++++++++++++ .../elasticsearch/end_to_end/RequestTest.java | 88 +++----------- .../spec_issues/SpecIssuesTest.java | 78 ++++++++++++ .../spec_issues/issue-0057-response.json | 62 ++++++++++ .../elasticsearch/spec_issues/issue-0078.json | 14 +++ .../spec_issues/issue-0107-response.json | 52 ++++++++ 7 files changed, 393 insertions(+), 74 deletions(-) create mode 100644 java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java create mode 100644 java-client/src/test/java/co/elastic/clients/elasticsearch/spec_issues/SpecIssuesTest.java create mode 100644 java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0057-response.json create mode 100644 java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0078.json create mode 100644 java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0107-response.json diff --git a/java-client/src/main/java/co/elastic/clients/json/ExternallyTaggedUnion.java b/java-client/src/main/java/co/elastic/clients/json/ExternallyTaggedUnion.java index 6321a0e5d..e83ebb771 100644 --- a/java-client/src/main/java/co/elastic/clients/json/ExternallyTaggedUnion.java +++ b/java-client/src/main/java/co/elastic/clients/json/ExternallyTaggedUnion.java @@ -24,8 +24,10 @@ import jakarta.json.stream.JsonParser; import jakarta.json.stream.JsonParsingException; +import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.BiFunction; @@ -57,13 +59,13 @@ public Deserializer(Map> deserialize /** * Deserialize a union value, given its type. */ - public Union deserialize(String type, JsonParser parser, JsonpMapper mapper) { + public Union deserialize(String type, JsonParser parser, JsonpMapper mapper, Event event) { JsonpDeserializer deserializer = deserializers.get(type); if (deserializer == null) { throw new JsonParsingException("Unknown variant type '" + type + "'", parser.getLocation()); } - return unionCtor.apply(type, deserializer.deserialize(parser, mapper)); + return unionCtor.apply(type, deserializer.deserialize(parser, mapper, event)); } /** @@ -104,10 +106,44 @@ public void deserializeEntry(String key, JsonParser parser, JsonpMapper mapper, String type = key.substring(0, hashPos); String name = key.substring(hashPos + 1); - targetMap.put(name, deserializer.deserialize(type, parser, mapper)); + targetMap.put(name, deserializer.deserialize(type, parser, mapper, parser.next())); } } + static > JsonpDeserializer>> arrayMapDeserializer( + TypedKeysDeserializer deserializer + ) { + return JsonpDeserializer.of( + EnumSet.of(Event.START_OBJECT), + (parser, mapper, event) -> { + Map> result = new HashMap<>(); + while ((event = parser.next()) != Event.END_OBJECT) { + JsonpUtils.expectEvent(parser, event, Event.KEY_NAME); + // Split key and type + String key = parser.getString(); + int hashPos = key.indexOf('#'); + if (hashPos == -1) { + throw new JsonParsingException( + "Property name '" + key + "' is not in the 'type#name' format. Make sure the request has 'typed_keys' set.", + parser.getLocation() + ); + } + + String type = key.substring(0, hashPos); + String name = key.substring(hashPos + 1); + + List list = new ArrayList<>(); + JsonpUtils.expectNextEvent(parser, Event.START_ARRAY); + while ((event = parser.next()) != Event.END_ARRAY) { + list.add(deserializer.deserializer.deserialize(type, parser, mapper, event)); + } + result.put(name, list); + } + return result; + } + ); + } + /** * Serialize an externally tagged union using the typed keys encoding. */ @@ -119,6 +155,26 @@ public void deserializeEntry(String key, JsonParser parser, JsonpMapper mapper, generator.writeEnd(); } + static > void serializeTypedKeysArray( + Map> map, JsonGenerator generator, JsonpMapper mapper + ) { + generator.writeStartObject(); + for (Map.Entry> entry: map.entrySet()) { + List list = entry.getValue(); + if (list.isEmpty()) { + continue; // We can't know the kind, skip this entry + } + + generator.writeKey(list.get(0)._kind().jsonValue() + "#" + entry.getKey()); + generator.writeStartArray(); + for (T value: list) { + value.serialize(generator, mapper); + } + generator.writeEnd(); + } + generator.writeEnd(); + } + /** * Serialize an externally tagged union using the typed keys encoding, without the enclosing start/end object. */ diff --git a/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java b/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java new file mode 100644 index 000000000..56a4bc9e1 --- /dev/null +++ b/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java @@ -0,0 +1,111 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.elasticsearch; + +import co.elastic.clients.json.JsonpMapper; +import co.elastic.clients.json.jsonb.JsonbJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.client.RestClient; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import java.time.Duration; + +public class ElasticsearchTestServer implements AutoCloseable { + + private volatile ElasticsearchContainer container; + private int port; + private final JsonpMapper mapper = new JsonbJsonpMapper(); + private RestClient restClient; + private ElasticsearchTransport transport; + private ElasticsearchClient client; + + private static ElasticsearchTestServer global; + + public static synchronized ElasticsearchTestServer global() { + if (global == null) { + System.out.println("Starting global ES test server."); + global = new ElasticsearchTestServer(); + global.setup(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + System.out.println("Stopping global ES test server."); + global.close(); + })); + } + return global; + } + + private synchronized void setup() { + container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2") + .withEnv("ES_JAVA_OPTS", "-Xms256m -Xmx256m") + .withEnv("path.repo", "/tmp") // for snapshots + .withStartupTimeout(Duration.ofSeconds(30)) + .withPassword("changeme"); + container.start(); + port = container.getMappedPort(9200); + + BasicCredentialsProvider credsProv = new BasicCredentialsProvider(); + credsProv.setCredentials( + AuthScope.ANY, new UsernamePasswordCredentials("elastic", "changeme") + ); + restClient = RestClient.builder(new HttpHost("localhost", port)) + .setHttpClientConfigCallback(hc -> hc.setDefaultCredentialsProvider(credsProv)) + .build(); + transport = new RestClientTransport(restClient, mapper); + client = new ElasticsearchClient(transport); + } + + @Override + public void close() { + if (this == global) { + // Closed with a shutdown hook + return; + } + + if (container != null) { + container.stop(); + } + container = null; + } + + public int port() { + return port; + } + + public RestClient restClient() { + return restClient; + } + + public ElasticsearchTransport transport() { + return transport; + } + + public JsonpMapper mapper() { + return mapper; + } + + public ElasticsearchClient client() { + return client; + } +} diff --git a/java-client/src/test/java/co/elastic/clients/elasticsearch/end_to_end/RequestTest.java b/java-client/src/test/java/co/elastic/clients/elasticsearch/end_to_end/RequestTest.java index 274ce3c16..0718ffc20 100644 --- a/java-client/src/test/java/co/elastic/clients/elasticsearch/end_to_end/RequestTest.java +++ b/java-client/src/test/java/co/elastic/clients/elasticsearch/end_to_end/RequestTest.java @@ -21,6 +21,7 @@ import co.elastic.clients.elasticsearch.ElasticsearchAsyncClient; import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch.ElasticsearchTestServer; import co.elastic.clients.elasticsearch._types.ElasticsearchException; import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch._types.aggregations.HistogramAggregate; @@ -42,26 +43,12 @@ import co.elastic.clients.elasticsearch.indices.GetMappingResponse; import co.elastic.clients.elasticsearch.indices.IndexState; import co.elastic.clients.elasticsearch.model.ModelTestCase; -import co.elastic.clients.elasticsearch.snapshot.CreateRepositoryResponse; -import co.elastic.clients.elasticsearch.snapshot.CreateSnapshotResponse; -import co.elastic.clients.json.JsonpMapper; -import co.elastic.clients.json.jsonb.JsonbJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.endpoints.BooleanResponse; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.testcontainers.elasticsearch.ElasticsearchContainer; import java.io.IOException; -import java.time.Duration; import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -70,38 +57,11 @@ public class RequestTest extends Assert { - private static ElasticsearchContainer container; - private static final JsonpMapper mapper = new JsonbJsonpMapper(); - private static RestClient restClient; - private static ElasticsearchTransport transport; - private static ElasticsearchClient client; + static ElasticsearchClient client; @BeforeClass public static void setup() { - container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2") - .withEnv("ES_JAVA_OPTS", "-Xms256m -Xmx256m") - .withEnv("path.repo", "/tmp") // for snapshots - .withStartupTimeout(Duration.ofSeconds(30)) - .withPassword("changeme"); - container.start(); - int port = container.getMappedPort(9200); - - BasicCredentialsProvider credsProv = new BasicCredentialsProvider(); - credsProv.setCredentials( - AuthScope.ANY, new UsernamePasswordCredentials("elastic", "changeme") - ); - restClient = RestClient.builder(new HttpHost("localhost", port)) - .setHttpClientConfigCallback(hc -> hc.setDefaultCredentialsProvider(credsProv)) - .build(); - transport = new RestClientTransport(restClient, mapper); - client = new ElasticsearchClient(transport); - } - - @AfterClass - public static void tearDown() { - if (container != null) { - container.stop(); - } + client = ElasticsearchTestServer.global().client(); } @Test @@ -112,7 +72,7 @@ public void testCount() throws Exception { @Test public void testIndexCreation() throws Exception { - ElasticsearchAsyncClient asyncClient = new ElasticsearchAsyncClient(transport); + ElasticsearchAsyncClient asyncClient = new ElasticsearchAsyncClient(client._transport()); // Ping the server assertTrue(client.ping().value()); @@ -222,7 +182,7 @@ public void testDataIngestion() throws Exception { public void testCatRequest() throws IOException { // Cat requests should have the "format=json" added by the transport NodesResponse nodes = client.cat().nodes(_0 -> _0); - System.out.println(ModelTestCase.toJson(nodes, mapper)); + System.out.println(ModelTestCase.toJson(nodes, client._transport().jsonpMapper())); assertEquals(1, nodes.valueBody().size()); assertEquals("*", nodes.valueBody().get(0).master()); @@ -247,15 +207,25 @@ public void testBulkRequest() throws IOException { .id("def") .document(appData) )) + .operations(_1 -> _1 + .update(_2 -> _2 + .index("foo") + .id("gh") + .action(_3 -> _3 + .docAsUpsert(true) + .doc(appData)) + ) + ) ); assertFalse(bulk.errors()); - assertEquals(2, bulk.items().size()); + assertEquals(3, bulk.items().size()); assertEquals(OperationType.Create, bulk.items().get(0).operationType()); assertEquals("foo", bulk.items().get(0).index()); assertEquals(1L, bulk.items().get(0).version().longValue()); assertEquals("foo", bulk.items().get(1).index()); assertEquals(1L, bulk.items().get(1).version().longValue()); + assertEquals(42, client.get(b -> b.index("foo").id("gh"), AppData.class).source().intValue); } @Test @@ -291,7 +261,7 @@ public void testRefresh() throws IOException { ExecutionException ee = assertThrows(ExecutionException.class, () -> { - ElasticsearchAsyncClient aClient = new ElasticsearchAsyncClient(transport); + ElasticsearchAsyncClient aClient = new ElasticsearchAsyncClient(client._transport()); GetResponse response = aClient.get( _0 -> _0.index("doesnotexist").id("reallynot"), String.class ).get(); @@ -398,30 +368,6 @@ public void testDefaultIndexSettings() throws IOException { assertNull(settings.get(index).defaults()); } - @Test - public void testSnapshotCreation() throws IOException { - // https://github.com/elastic/elasticsearch-java/issues/74 - // https://github.com/elastic/elasticsearch/issues/82358 - - CreateRepositoryResponse repo = client.snapshot().createRepository(b1 -> b1 - .name("test") - .type("fs") - .settings(b2 -> b2 - .location("/tmp/test-repo") - ) - ); - - assertTrue(repo.acknowledged()); - - CreateSnapshotResponse snapshot = client.snapshot().create(b -> b - .repository("test") - .snapshot("1") - .waitForCompletion(true) - ); - - assertNotNull(snapshot.snapshot()); - } - @Test public void testValueBodyResponse() throws Exception { DiskUsageResponse resp = client.indices().diskUsage(b -> b diff --git a/java-client/src/test/java/co/elastic/clients/elasticsearch/spec_issues/SpecIssuesTest.java b/java-client/src/test/java/co/elastic/clients/elasticsearch/spec_issues/SpecIssuesTest.java new file mode 100644 index 000000000..a9427ed45 --- /dev/null +++ b/java-client/src/test/java/co/elastic/clients/elasticsearch/spec_issues/SpecIssuesTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.elasticsearch.spec_issues; + +import co.elastic.clients.elasticsearch.ElasticsearchTestServer; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.model.ModelTestCase; +import co.elastic.clients.json.JsonData; +import co.elastic.clients.json.JsonpDeserializer; +import jakarta.json.stream.JsonParser; +import org.junit.Test; + +import java.io.InputStream; + +/** + * Test issues related to the API specifications. + * + * Depending on the feedback provided, this may involve either loading a JSON file or sending requests to an ES server. + */ +public class SpecIssuesTest extends ModelTestCase { + + @Test + public void i0107_rangeBucketKey() { + // https://github.com/elastic/elasticsearch-java/issues/107 + loadRsrc("issue-0107-response.json", SearchResponse.createSearchResponseDeserializer(JsonData._DESERIALIZER)); + } + + @Test + public void i0078_deserializeSearchRequest() { + // https://github.com/elastic/elasticsearch-java/issues/78 + loadRsrc("issue-0078.json", SearchRequest._DESERIALIZER); + } + + @Test + public void i0057_suggestDeserialization() { + // https://github.com/elastic/elasticsearch-java/issues/57 + // Note: the _type properties have been removed so that the test works in 8.x too. + SearchResponse resp = loadRsrc("issue-0057-response.json", + SearchResponse.createSearchResponseDeserializer(JsonData._DESERIALIZER)); + + assertEquals(1, resp.suggest().get("completion:completion1").size()); + assertEquals("hash", resp.suggest().get("completion:completion1").get(0).completion().text()); + assertEquals("HashMap-Complete1", resp.suggest().get("completion:completion1").get(0).completion().options().get(0).text()); + } + + @Test + public void i0056_hitsMetadataTotal() throws Exception { + // https://github.com/elastic/elasticsearch-java/issues/56 + SearchResponse res = ElasticsearchTestServer.global().client() + .search(srb -> srb + .trackTotalHits(thb -> thb.enabled(false)), JsonData.class); + } + + private T loadRsrc(String res, JsonpDeserializer deser) { + InputStream is = this.getClass().getResourceAsStream(res); + assertNotNull("Resource not found: " + res, is); + JsonParser parser = mapper.jsonProvider().createParser(is); + return deser.deserialize(parser, mapper); + } +} diff --git a/java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0057-response.json b/java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0057-response.json new file mode 100644 index 000000000..3c69b89e9 --- /dev/null +++ b/java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0057-response.json @@ -0,0 +1,62 @@ +{ + "took": 67, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": null, + "hits": [] + }, + "suggest": { + "completion#completion:completion1": [ + { + "text": "hash", + "offset": 0, + "length": 4, + "options": [ + { + "text": "HashMap-Complete1", + "_index": "document", + "_id": "2", + "_score": 1.0 + }, + { + "text": "HashSet-Complete1", + "_index": "document", + "_id": "1", + "_score": 1.0 + } + ] + } + ], + "completion#completion:completion2": [ + { + "text": "hash", + "offset": 0, + "length": 4, + "options": [ + { + "text": "HashMap-Complete2", + "_index": "document", + "_id": "2", + "_score": 1.0 + }, + { + "text": "HashSet-Complete2", + "_index": "document", + "_id": "1", + "_score": 1.0 + } + ] + } + ] + } +} diff --git a/java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0078.json b/java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0078.json new file mode 100644 index 000000000..506b42b15 --- /dev/null +++ b/java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0078.json @@ -0,0 +1,14 @@ +{ + "size": 9999, + "query": { + "match_all": {} + }, + "sort": [ + { + "modify_time": { + "order": "desc" + } + } + ], + "track_total_hits": 2147483647 +} diff --git a/java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0107-response.json b/java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0107-response.json new file mode 100644 index 000000000..b76023b52 --- /dev/null +++ b/java-client/src/test/resources/co/elastic/clients/elasticsearch/spec_issues/issue-0107-response.json @@ -0,0 +1,52 @@ +{ + "took" : 1, + "timed_out" : false, + "_shards" : { + "total" : 1, + "successful" : 1, + "skipped" : 0, + "failed" : 0 + }, + "hits" : { + "total" : { + "value" : 5, + "relation" : "eq" + }, + "max_score" : null, + "hits" : [ ] + }, + "aggregations": { + "date_range#date_ranges": { + "buckets": [ + { + "key": "2Wk", + "from": 1.6408224E12, + "from_as_string": "2021-12-30T00:00:00.000Z", + "to": 1.642032E12, + "to_as_string": "2022-01-06T00:00:00.000Z", + "doc_count": 0, + "avg#avgCost": { + "value": null + }, + "cardinality#uniqueUsers": { + "value": 0 + } + }, + { + "key": "1Wk", + "from": 1.6414272E12, + "from_as_string": "2022-01-06T00:00:00.000Z", + "to": 1.642032E12, + "to_as_string": "2022-01-13T00:00:00.000Z", + "doc_count": 0, + "avg#avgCost": { + "value": null + }, + "cardinality#uniqueUsers": { + "value": 0 + } + } + ] + } + } +}