From 5ecc6c7fb0a129382acd6b57bf97afce3cca0483 Mon Sep 17 00:00:00 2001 From: Maksim Strutovskii Date: Mon, 5 Jun 2023 14:21:58 +0300 Subject: [PATCH] fix: add missing translog sync interval option to index settings Additionally, server response contains separate `translog` object, which cannot be deserialized by present translog option deserializers. Therefore, this commit introduces a corresponding `Translog` class, similarly to how it is done for the `blocks` property. Signed-off-by: Maksim Strutovskii --- CHANGELOG.md | 2 + .../opensearch/indices/IndexSettings.java | 59 ++++++ .../client/opensearch/indices/Translog.java | 170 ++++++++++++++++++ .../opensearch/experiments/ParsingTests.java | 74 +++++--- .../integTest/AbstractRequestIT.java | 10 +- 5 files changed, 288 insertions(+), 27 deletions(-) create mode 100644 java-client/src/main/java/org/opensearch/client/opensearch/indices/Translog.java diff --git a/CHANGELOG.md b/CHANGELOG.md index fb4dd19fe0..9f17e3f5eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fix failure when deserialing response for tasks API ([#463](https://github.com/opensearch-project/opensearch-java/pull/463)) - Fix failure when deserializing boolean types for enums ([#463](https://github.com/opensearch-project/opensearch-java/pull/482)) - Fix missing minScore, postFilter, searchAfter, sort, trackScores in the MultisearchBody ([#516](https://github.com/opensearch-project/opensearch-java/pull/516)) +- Fix missing translog sync interval option in index settings ([#518](https://github.com/opensearch-project/opensearch-java/pull/518)) + ### Security ## [2.4.0] - 04/11/2023 diff --git a/java-client/src/main/java/org/opensearch/client/opensearch/indices/IndexSettings.java b/java-client/src/main/java/org/opensearch/client/opensearch/indices/IndexSettings.java index bae7d418cb..d732096e1e 100644 --- a/java-client/src/main/java/org/opensearch/client/opensearch/indices/IndexSettings.java +++ b/java-client/src/main/java/org/opensearch/client/opensearch/indices/IndexSettings.java @@ -198,12 +198,18 @@ public class IndexSettings implements JsonpSerializable { @Nullable private final Integer maxSlicesPerScroll; + @Nullable + private final Translog translog; + @Nullable private final String translogDurability; @Nullable private final String translogFlushThresholdSize; + @Nullable + private final Time translogSyncInterval; + @Nullable private final Boolean queryStringLenient; @@ -274,8 +280,10 @@ private IndexSettings(Builder builder) { this.verifiedBeforeClose = builder.verifiedBeforeClose; this.format = builder.format; this.maxSlicesPerScroll = builder.maxSlicesPerScroll; + this.translog = builder.translog; this.translogDurability = builder.translogDurability; this.translogFlushThresholdSize = builder.translogFlushThresholdSize; + this.translogSyncInterval = builder.translogSyncInterval; this.queryStringLenient = builder.queryStringLenient; this.priority = builder.priority; this.topMetricsMaxSize = builder.topMetricsMaxSize; @@ -672,6 +680,14 @@ public final Integer maxSlicesPerScroll() { return this.maxSlicesPerScroll; } + /** + * API name: {@code translog} + */ + @Nullable + public final Translog translog() { + return this.translog; + } + /** * API name: {@code translog.durability} */ @@ -688,6 +704,14 @@ public final String translogFlushThresholdSize() { return this.translogFlushThresholdSize; } + /** + * API name: {@code translog.sync_interval} + */ + @Nullable + public final Time translogSyncInterval() { + return this.translogSyncInterval; + } + /** * API name: {@code query_string.lenient} */ @@ -991,6 +1015,11 @@ protected void serializeInternal(JsonGenerator generator, JsonpMapper mapper) { generator.writeKey("max_slices_per_scroll"); generator.write(this.maxSlicesPerScroll); + } + if (this.translog != null) { + generator.writeKey("translog"); + this.translog.serialize(generator, mapper); + } if (this.translogDurability != null) { generator.writeKey("translog.durability"); @@ -1001,6 +1030,11 @@ protected void serializeInternal(JsonGenerator generator, JsonpMapper mapper) { generator.writeKey("translog.flush_threshold_size"); generator.write(this.translogFlushThresholdSize); + } + if (this.translogSyncInterval != null) { + generator.writeKey("translog.sync_interval"); + this.translogSyncInterval.serialize(generator, mapper); + } if (this.queryStringLenient != null) { generator.writeKey("query_string.lenient"); @@ -1186,12 +1220,18 @@ public static class Builder extends ObjectBuilderBase implements ObjectBuilder> fn) { + return fn.apply(new Builder()).build(); + } + + /** + * API name: {@code durability} + */ + @Nullable + public final String durability() { + return this.durability; + } + + /** + * API name: {@code flush_threshold_size} + */ + @Nullable + public final String flushThresholdSize() { + return this.flushThresholdSize; + } + + /** + * API name: {@code sync_interval} + */ + @Nullable + public final Time syncInterval() { + return this.syncInterval; + } + + /** + * Serialize this object to JSON. + */ + public void serialize(JsonGenerator generator, JsonpMapper mapper) { + generator.writeStartObject(); + serializeInternal(generator, mapper); + generator.writeEnd(); + } + + protected void serializeInternal(JsonGenerator generator, JsonpMapper mapper) { + + if (this.durability != null) { + generator.writeKey("durability"); + generator.write(this.durability); + + } + if (this.flushThresholdSize != null) { + generator.writeKey("flush_threshold_size"); + generator.write(this.flushThresholdSize); + + } + if (this.syncInterval != null) { + generator.writeKey("sync_interval"); + this.syncInterval.serialize(generator, mapper); + + } + + } + + // --------------------------------------------------------------------------------------------- + + /** + * Builder for {@link Translog}. + */ + public static class Builder extends ObjectBuilderBase implements ObjectBuilder { + + @Nullable + private String durability; + + @Nullable + private String flushThresholdSize; + + @Nullable + private Time syncInterval; + + /** + * API name: {@code durability} + */ + public final Translog.Builder durability(@Nullable String value) { + this.durability = value; + return this; + } + + /** + * API name: {@code flush_threshold_size} + */ + public final Translog.Builder flushThresholdSize(@Nullable String value) { + this.flushThresholdSize = value; + return this; + } + + /** + * API name: {@code sync_interval} + */ + public final Translog.Builder syncInterval(@Nullable Time value) { + this.syncInterval = value; + return this; + } + + /** + * Builds a {@link Translog}. + * + * @throws NullPointerException + * if some of the required fields are null. + */ + public Translog build() { + _checkSingleUse(); + + return new Translog(this); + } + } + + /** + * Json deserializer for {@link Translog} + */ + public static final JsonpDeserializer _DESERIALIZER = ObjectBuilderDeserializer.lazy(Builder::new, + Translog::setupTranslogDeserializer); + + protected static void setupTranslogDeserializer(ObjectDeserializer op) { + + op.add(Translog.Builder::durability, JsonpDeserializer.stringDeserializer(), "durability"); + op.add(Translog.Builder::flushThresholdSize, JsonpDeserializer.stringDeserializer(), "flush_threshold_size"); + op.add(Translog.Builder::syncInterval, Time._DESERIALIZER, "sync_interval"); + + } + +} diff --git a/java-client/src/test/java/org/opensearch/client/opensearch/experiments/ParsingTests.java b/java-client/src/test/java/org/opensearch/client/opensearch/experiments/ParsingTests.java index 6bbf1917a8..1e332d8bb5 100644 --- a/java-client/src/test/java/org/opensearch/client/opensearch/experiments/ParsingTests.java +++ b/java-client/src/test/java/org/opensearch/client/opensearch/experiments/ParsingTests.java @@ -38,8 +38,12 @@ import jakarta.json.stream.JsonParsingException; import org.junit.Assert; import org.junit.Test; +import org.opensearch.client.json.JsonpDeserializer; +import org.opensearch.client.json.JsonpSerializable; import org.opensearch.client.json.jsonb.JsonbJsonpMapper; +import org.opensearch.client.opensearch._types.Time; import org.opensearch.client.opensearch.experiments.api.FooRequest; +import org.opensearch.client.opensearch.indices.IndexSettings; import org.opensearch.client.opensearch.indices.IndexSettingsMapping; import java.io.ByteArrayInputStream; @@ -53,8 +57,6 @@ public void testFoo() throws Exception { try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - FooRequest foo = FooRequest.builder() .name("z") .value(1) @@ -64,19 +66,10 @@ public void testFoo() throws Exception { ) .build(); - JsonProvider provider = JsonProvider.provider(); - JsonGenerator generator = provider.createGenerator(baos); - foo.serialize(generator, new JsonbJsonpMapper()); - - generator.close(); - String str = baos.toString(); - + String str = serialize(foo); assertEquals("{\"name\":\"z\",\"value\":1,\"indices\":[\"a\",\"b\",\"c\"],\"bar\":{\"name\":\"Raise the bar\"}}", str); - JsonParser parser = provider.createParser(new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))); - - FooRequest foo2 = FooRequest.parser().deserialize(parser, new JsonbJsonpMapper()); - + FooRequest foo2 = deserialize(str, FooRequest.parser()); assertEquals(foo.name(), foo2.name()); assertEquals(foo.value(), foo2.value()); assertNull(foo2.size()); @@ -88,11 +81,31 @@ public void testFoo() throws Exception { } @Test - public void testIndexSettingsMappingParsing() { + public void testIndexSettingsSyncIntervalTimeParsing() { try { - var baos = new ByteArrayOutputStream(); + var indexSettings = IndexSettings.of(_1 -> _1.translogSyncInterval(Time.of(_2 -> _2.time("10s")))); + + var str = serialize(indexSettings); + assertEquals("{\"translog.sync_interval\":\"10s\"}", str); + + IndexSettings deserialized = deserialize(str, IndexSettings._DESERIALIZER); + assertEquals(indexSettings.translogSyncInterval().time(), deserialized.translogSyncInterval().time()); + + var responseStr = "{\"translog\":{\"sync_interval\":\"10s\"}}"; + IndexSettings responseDeserialized = deserialize(responseStr, IndexSettings._DESERIALIZER); + assertEquals(indexSettings.translogSyncInterval().time(), responseDeserialized.translog().syncInterval().time()); + + } catch (JsonParsingException je) { + throw new JsonParsingException(je.getMessage() + " at " + je.getLocation(), je, je.getLocation()); + } + } + + @Test + public void testIndexSettingsMappingParsing() { + + try { var mapping = IndexSettingsMapping.of(b -> b .totalFields(d -> d.limit(1L)) @@ -101,20 +114,11 @@ public void testIndexSettingsMappingParsing() { .nestedObjects(d -> d.limit(4L)) .fieldNameLength(d -> d.limit(5L))); - var provider = JsonProvider.provider(); - var generator = provider.createGenerator(baos); - mapping.serialize(generator, new JsonbJsonpMapper()); - - generator.close(); - var str = baos.toString(); - + var str = serialize(mapping); assertEquals("{\"total_fields\":{\"limit\":1},\"depth\":{\"limit\":2},\"nested_fields\":{\"limit\":3}," + "\"nested_objects\":{\"limit\":4},\"field_name_length\":{\"limit\":5}}", str); - var parser = provider.createParser(new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))); - - var deserialized = IndexSettingsMapping._DESERIALIZER.deserialize(parser, new JsonbJsonpMapper()); - + var deserialized = deserialize(str, IndexSettingsMapping._DESERIALIZER); assertEquals(mapping.totalFields().limit(), deserialized.totalFields().limit()); assertEquals(mapping.depth().limit(), deserialized.depth().limit()); assertEquals(mapping.nestedFields().limit(), deserialized.nestedFields().limit()); @@ -124,4 +128,22 @@ public void testIndexSettingsMappingParsing() { throw new JsonParsingException(je.getMessage() + " at " + je.getLocation(), je, je.getLocation()); } } + + private T deserialize(String serializedValue, JsonpDeserializer deserializer) { + var provider = JsonProvider.provider(); + var parser = provider.createParser(new ByteArrayInputStream(serializedValue.getBytes(StandardCharsets.UTF_8))); + + return deserializer.deserialize(parser, new JsonbJsonpMapper()); + } + + private String serialize(JsonpSerializable object) { + var baos = new ByteArrayOutputStream(); + var provider = JsonProvider.provider(); + + var generator = provider.createGenerator(baos); + object.serialize(generator, new JsonbJsonpMapper()); + generator.close(); + + return baos.toString(); + } } diff --git a/java-client/src/test/java/org/opensearch/client/opensearch/integTest/AbstractRequestIT.java b/java-client/src/test/java/org/opensearch/client/opensearch/integTest/AbstractRequestIT.java index 0435cb9f72..376ab63986 100644 --- a/java-client/src/test/java/org/opensearch/client/opensearch/integTest/AbstractRequestIT.java +++ b/java-client/src/test/java/org/opensearch/client/opensearch/integTest/AbstractRequestIT.java @@ -114,10 +114,11 @@ public void testIndexCreation() throws Exception { } @Test - public void testIndexSettingsMapping() throws Exception { + public void testIndexSettings() throws Exception { var createResponse = javaClient().indices() .create(r -> r.index("test-settings") .settings(s -> s + .translogSyncInterval(Time.of(t -> t.time("10s"))) .mapping(m -> m .fieldNameLength(f -> f.limit(300L)) .totalFields(f -> f.limit(30L)) @@ -133,6 +134,9 @@ public void testIndexSettingsMapping() throws Exception { assertNotNull(createdIndexSettings); assertNotNull(createdIndexSettings.settings()); assertNotNull(createdIndexSettings.settings().index()); + assertNotNull(createdIndexSettings.settings().index().translog()); + assertNotNull(createdIndexSettings.settings().index().translog().syncInterval()); + assertEquals("10s", createdIndexSettings.settings().index().translog().syncInterval().time()); var createdMappingSettings = createdIndexSettings.settings().index().mapping(); assertNotNull(createdMappingSettings); assertNotNull(createdMappingSettings.fieldNameLength()); @@ -149,6 +153,7 @@ public void testIndexSettingsMapping() throws Exception { var putSettingsResponse = javaClient().indices().putSettings(r -> r .index("test-settings") .settings(s -> s + .translogSyncInterval(Time.of(t -> t.time("5s"))) .mapping(m -> m .fieldNameLength(f -> f.limit(400L)) .totalFields(f -> f.limit(130L)) @@ -163,6 +168,9 @@ public void testIndexSettingsMapping() throws Exception { assertNotNull(updatedIndexSettings); assertNotNull(updatedIndexSettings.settings()); assertNotNull(updatedIndexSettings.settings().index()); + assertNotNull(updatedIndexSettings.settings().index().translog()); + assertNotNull(updatedIndexSettings.settings().index().translog().syncInterval()); + assertEquals("5s", updatedIndexSettings.settings().index().translog().syncInterval().time()); var updatedMappingSettings = updatedIndexSettings.settings().index().mapping(); assertNotNull(updatedMappingSettings); assertNotNull(updatedMappingSettings.fieldNameLength());